Abacn commented on code in PR #39154:
URL: https://github.com/apache/beam/pull/39154#discussion_r3494628355
##########
.test-infra/dataproc/flink_cluster.sh:
##########
@@ -135,20 +147,17 @@ function create_cluster() {
if [[ -n "${HIGH_MEM_MACHINE:=}" ]]; then
worker_machine_type="${HIGH_MEM_MACHINE}"
master_machine_type="${HIGH_MEM_MACHINE}"
-
- gcloud dataproc clusters create $CLUSTER_NAME --enable-component-gateway
--region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \
- --master-machine-type=${master_machine_type}
--worker-machine-type=${worker_machine_type} --metadata "${metadata}", \
- --image-version=$image_version --zone=$GCLOUD_ZONE
--optional-components=FLINK,DOCKER \
- --properties="${HIGH_MEM_FLINK_PROPS}"
- else
- # Docker init action restarts yarn so we need to start yarn session after
this restart happens.
- # This is why flink init action is invoked last.
- # TODO(11/22/2024) remove --worker-machine-type and --master-machine-type
once N2 CPUs quota relaxed
- # Dataproc 2.1 uses n2-standard-2 by default but there is N2 CPUs=24 quota
limit for this project
- gcloud dataproc clusters create $CLUSTER_NAME --enable-component-gateway
--region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \
- --master-machine-type=${master_machine_type}
--worker-machine-type=${worker_machine_type} --metadata "${metadata}", \
- --image-version=$image_version --zone=$GCLOUD_ZONE
--optional-components=FLINK,DOCKER --quiet
fi
+
+ # Docker init action restarts yarn so we need to start yarn session after
this restart happens.
+ # This is why flink init action is invoked last.
+ # TODO(11/22/2024) remove --worker-machine-type and --master-machine-type
once N2 CPUs quota relaxed
+ # Dataproc 2.1 uses n2-standard-2 by default but there is N2 CPUs=24 quota
limit for this project
+ gcloud dataproc clusters create $CLUSTER_NAME --enable-component-gateway
--region=$GCLOUD_REGION --num-workers=$FLINK_NUM_WORKERS --public-ip-address \
+ --master-machine-type=${master_machine_type}
--worker-machine-type=${worker_machine_type} --metadata "${metadata}", \
+ --image-version=$image_version --optional-components=FLINK,DOCKER \
+ --initialization-actions="${BEAM_INIT}" \
Review Comment:
`--initialization-actions` was not effective after past changes. Removing
them except image prepull in beam.sh. Set Flink properties directly in
flink_cluster.sh
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
