This is an automated email from the ASF dual-hosted git repository.
xqhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 60f87930a70 Use --enable-component-gateway when creating the flink
cluster (#33198)
60f87930a70 is described below
commit 60f87930a70b50f4978dc8abe7dd4724ad9d383e
Author: liferoad <[email protected]>
AuthorDate: Mon Nov 25 09:43:44 2024 -0500
Use --enable-component-gateway when creating the flink cluster (#33198)
* Use --enable-component-gateway when creating the flink cluster
* Update flink_cluster.sh
---
.test-infra/dataproc/flink_cluster.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/.test-infra/dataproc/flink_cluster.sh
b/.test-infra/dataproc/flink_cluster.sh
index aeb5ec19441..24d51ac13bd 100755
--- a/.test-infra/dataproc/flink_cluster.sh
+++ b/.test-infra/dataproc/flink_cluster.sh
@@ -131,8 +131,10 @@ function create_cluster() {
# Docker init action restarts yarn so we need to start yarn session after
this restart happens.
# This is why flink init action is invoked last.
- gcloud dataproc clusters create $CLUSTER_NAME --region=$GCLOUD_REGION
--num-workers=$FLINK_NUM_WORKERS --public-ip-address \
- --metadata "${metadata}", \
+ # TODO(11/22/2024) remove --worker-machine-type and --master-machine-type
once N2 CPUs quota relaxed
+ # Dataproc 2.1 uses n2-standard-2 by default but there is N2 CPUs=24 quota
limit for this project
+ gcloud dataproc clusters create $CLUSTER_NAME --enable-component-gateway
--region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \
+ --master-machine-type=n1-standard-2 --worker-machine-type=n1-standard-2
--metadata "${metadata}", \
--image-version=$image_version --zone=$GCLOUD_ZONE
--optional-components=FLINK,DOCKER --quiet
}
