[airflow] branch main updated: fix Dataproc system tests (#34307)

[eladkal]

This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 55c704bb54 fix Dataproc system tests (#34307)
55c704bb54 is described below

commit 55c704bb54dd2348a21f948f561177983dd89324
Author: max <42827971+moiseen...@users.noreply.github.com>
AuthorDate: Tue Sep 12 17:49:02 2023 +0200

    fix Dataproc system tests (#34307)
---
 .../cloud/dataproc/example_dataproc_cluster_generator.py    |  3 +--
 .../providers/google/cloud/dataproc/example_dataproc_gke.py | 13 +++----------
 .../cloud/dataproc/example_dataproc_workflow_deferrable.py  |  4 ++--
 .../google/cloud/dataproc/resources/pip-install.sh          |  3 ++-
 4 files changed, 8 insertions(+), 15 deletions(-)

diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
index 746ee9df0d..9267dbf7eb 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
@@ -114,9 +114,8 @@ with models.DAG(
         # TEST SETUP
         create_bucket
         >> upload_file
-        >>
         # TEST BODY
-        create_dataproc_cluster
+        >> create_dataproc_cluster
         # TEST TEARDOWN
         >> [delete_cluster, delete_bucket]
     )
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py
index b5451d0d44..2958229a15 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_gke.py
@@ -31,7 +31,6 @@ import os
 from datetime import datetime
 
 from airflow import models
-from airflow.operators.bash import BashOperator
 from airflow.providers.google.cloud.operators.dataproc import (
     DataprocCreateClusterOperator,
     DataprocDeleteClusterOperator,
@@ -98,13 +97,6 @@ with models.DAG(
         body=GKE_CLUSTER_CONFIG,
     )
 
-    add_iam_policy_binding = BashOperator(
-        task_id="add_iam_policy_binding",
-        bash_command=f"gcloud projects add-iam-policy-binding {PROJECT_ID} "
-        f"--member=serviceAccount:{WORKLOAD_POOL}[{GKE_NAMESPACE}/agent] "
-        "--role=roles/iam.workloadIdentityUser",
-    )
-
     # [START how_to_cloud_dataproc_create_cluster_operator_in_gke]
     create_cluster_in_gke = DataprocCreateClusterOperator(
         task_id="create_cluster_in_gke",
@@ -134,11 +126,12 @@ with models.DAG(
     (
         # TEST SETUP
         create_gke_cluster
-        >> add_iam_policy_binding
         # TEST BODY
         >> create_cluster_in_gke
         # TEST TEARDOWN
-        >> [delete_gke_cluster, delete_dataproc_cluster]
+        # >> delete_gke_cluster
+        >> delete_dataproc_cluster
+        >> delete_gke_cluster
     )
 
     from tests.system.utils.watcher import watcher
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
index 3e03e0e884..c6f9909d92 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
@@ -34,7 +34,7 @@ DAG_ID = "dataproc_workflow_def"
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
 
 REGION = "europe-west1"
-CLUSTER_NAME = f"{ENV_ID}-{DAG_ID}".replace("_", "-")
+CLUSTER_NAME = f"cluster-{ENV_ID}-{DAG_ID}".replace("_", "")
 CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
@@ -48,7 +48,7 @@ CLUSTER_CONFIG = {
     },
 }
 PIG_JOB = {"query_list": {"queries": ["define sin HiveUDF('sin');"]}}
-WORKFLOW_NAME = "airflow-dataproc-test"
+WORKFLOW_NAME = "airflow-dataproc-test-def"
 WORKFLOW_TEMPLATE = {
     "id": WORKFLOW_NAME,
     "placement": {
diff --git 
a/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh 
b/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh
index ec25389e98..a9839b5b80 100644
--- a/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh
+++ b/tests/system/providers/google/cloud/dataproc/resources/pip-install.sh
@@ -62,7 +62,8 @@ function main() {
   fi
 
   install_pip
-  run_with_retry pip install --upgrade "${PACKAGES}"
+  # shellcheck disable=SC2086
+  run_with_retry pip install --upgrade ${PACKAGES}
 }
 
 main