This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 41503140fc Do not ignore the internal_ip_only if set to false in
Dataproc cluster config (#37014)
41503140fc is described below
commit 41503140fcb063e79d837443eeaf4ab7d7110cb4
Author: Ahzaz Hingora <[email protected]>
AuthorDate: Wed Jan 31 17:11:51 2024 +0530
Do not ignore the internal_ip_only if set to false in Dataproc cluster
config (#37014)
---
airflow/providers/google/cloud/operators/dataproc.py | 6 +++---
tests/providers/google/cloud/operators/test_dataproc.py | 13 +++++++++++++
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/airflow/providers/google/cloud/operators/dataproc.py
b/airflow/providers/google/cloud/operators/dataproc.py
index e66dd9d0e3..647f485ee1 100644
--- a/airflow/providers/google/cloud/operators/dataproc.py
+++ b/airflow/providers/google/cloud/operators/dataproc.py
@@ -361,10 +361,10 @@ class ClusterGenerator:
if self.subnetwork_uri:
cluster_data[config]["subnetwork_uri"] = self.subnetwork_uri
- if self.internal_ip_only:
- if not self.subnetwork_uri:
+ if self.internal_ip_only is not None:
+ if not self.subnetwork_uri and self.internal_ip_only:
raise AirflowException("Set internal_ip_only to true only when
you pass a subnetwork_uri.")
- cluster_data[config]["internal_ip_only"] = True
+ cluster_data[config]["internal_ip_only"] = self.internal_ip_only
if self.tags:
cluster_data[config]["tags"] = self.tags
diff --git a/tests/providers/google/cloud/operators/test_dataproc.py
b/tests/providers/google/cloud/operators/test_dataproc.py
index 686c2f10b2..68795357e7 100644
--- a/tests/providers/google/cloud/operators/test_dataproc.py
+++ b/tests/providers/google/cloud/operators/test_dataproc.py
@@ -677,6 +677,19 @@ class TestsClusterGenerator:
cluster = generator.make()
assert CONFIG_WITH_GPU_ACCELERATOR == cluster
+ def test_build_with_default_value_for_internal_ip_only(self):
+ generator = ClusterGenerator(project_id="project_id")
+ cluster = generator.make()
+ assert "internal_ip_only" not in cluster["gce_cluster_config"]
+
+ def test_build_sets_provided_value_for_internal_ip_only(self):
+ for internal_ip_only in [True, False]:
+ generator = ClusterGenerator(
+ project_id="project_id", internal_ip_only=internal_ip_only,
subnetwork_uri="subnetwork_uri"
+ )
+ cluster = generator.make()
+ assert cluster["gce_cluster_config"]["internal_ip_only"] ==
internal_ip_only
+
class TestDataprocCreateClusterOperator(DataprocClusterTestBase):
def test_deprecation_warning(self):
