This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0e157b38a3 Fix K8S executor override config using pod_override_object
(#35185)
0e157b38a3 is described below
commit 0e157b38a3e44b5a6fc084c581a025434a97a4c0
Author: Hussein Awala <[email protected]>
AuthorDate: Thu Nov 23 22:48:58 2023 +0200
Fix K8S executor override config using pod_override_object (#35185)
* Fix K8S executor override config using pod_override_object
* Activate multi namespace mode for K8S tests
* Force multi namespace for k8s tests
* Increase timeout to test
* Increase pytest execution-timeout
* Support setuping multiple worker namespaces in the helm chart
* Rollback chart changes
* Revert timeout increase
* Duplicate Airflow resources to test namespace
* Fix the commands used to duplicate the resources
---
airflow/providers/cncf/kubernetes/pod_generator.py | 4 +-
.../airflow_breeze/commands/kubernetes_commands.py | 56 ++++++++++++++++++++++
.../commands/kubernetes_commands_config.py | 1 +
.../src/airflow_breeze/utils/kubernetes_utils.py | 2 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 6 +--
images/breeze/output_k8s_deploy-airflow.svg | 50 ++++++++++---------
images/breeze/output_k8s_deploy-airflow.txt | 2 +-
.../cncf/kubernetes/test_pod_generator.py | 18 +++++--
8 files changed, 106 insertions(+), 33 deletions(-)
diff --git a/airflow/providers/cncf/kubernetes/pod_generator.py
b/airflow/providers/cncf/kubernetes/pod_generator.py
index 581aa61d16..69ca6201c4 100644
--- a/airflow/providers/cncf/kubernetes/pod_generator.py
+++ b/airflow/providers/cncf/kubernetes/pod_generator.py
@@ -434,8 +434,8 @@ class PodGenerator:
)
# Reconcile the pods starting with the first chronologically,
- # Pod from the pod_template_File -> Pod from executor_config arg ->
Pod from the K8s executor
- pod_list = [base_worker_pod, pod_override_object, dynamic_pod]
+ # Pod from the pod_template_File -> Pod from the K8s executor -> Pod
from executor_config arg
+ pod_list = [base_worker_pod, dynamic_pod, pod_override_object]
try:
pod = reduce(PodGenerator.reconcile_pods, pod_list)
diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
index c46d2262ad..d90a856d25 100644
--- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
@@ -167,6 +167,13 @@ option_use_standard_naming = click.option(
envvar="USE_STANDARD_NAMING",
)
+option_multi_namespace_mode = click.option(
+ "--multi-namespace-mode",
+ help="Use multi namespace mode.",
+ is_flag=True,
+ envvar="MULTI_NAMESPACE_MODE",
+)
+
option_rebuild_base_image = click.option(
"--rebuild-base-image",
help="Rebuilds base Airflow image before building K8S image.",
@@ -954,6 +961,7 @@ def _deploy_helm_chart(
executor: str,
use_standard_naming: bool,
extra_options: tuple[str, ...] | None = None,
+ multi_namespace_mode: bool = False,
) -> RunCommandResult:
cluster_name = get_kubectl_cluster_name(python=python,
kubernetes_version=kubernetes_version)
action = "Deploying" if not upgrade else "Upgrading"
@@ -995,6 +1003,8 @@ def _deploy_helm_chart(
"--set",
f"executor={executor}",
]
+ if multi_namespace_mode:
+ helm_command.extend(["--set", "multiNamespaceMode=true"])
if upgrade:
# force upgrade
helm_command.append("--force")
@@ -1024,6 +1034,7 @@ def _deploy_airflow(
wait_time_in_seconds: int,
use_standard_naming: bool,
extra_options: tuple[str, ...] | None = None,
+ multi_namespace_mode: bool = False,
) -> tuple[int, str]:
action = "Deploying" if not upgrade else "Upgrading"
cluster_name = get_kind_cluster_name(python=python,
kubernetes_version=kubernetes_version)
@@ -1036,8 +1047,47 @@ def _deploy_airflow(
executor=executor,
use_standard_naming=use_standard_naming,
extra_options=extra_options,
+ multi_namespace_mode=multi_namespace_mode,
)
if result.returncode == 0:
+ if multi_namespace_mode:
+ # duplicate Airflow configmaps, secrets and service accounts to
test namespace
+ run_command_with_k8s_env(
+ f"kubectl get secret -n {HELM_AIRFLOW_NAMESPACE} "
+ "--field-selector type!=helm.sh/release.v1 -o yaml "
+ f"| sed 's/namespace: {HELM_AIRFLOW_NAMESPACE}/namespace:
{TEST_NAMESPACE}/' "
+ f"| kubectl apply -n {TEST_NAMESPACE} -f -",
+ python=python,
+ kubernetes_version=kubernetes_version,
+ output=output,
+ check=False,
+ shell=True,
+ )
+
+ run_command_with_k8s_env(
+ f"kubectl get configmap -n {HELM_AIRFLOW_NAMESPACE} "
+ "--field-selector metadata.name!=kube-root-ca.crt -o yaml "
+ f"| sed 's/namespace: {HELM_AIRFLOW_NAMESPACE}/namespace:
{TEST_NAMESPACE}/' "
+ f"| kubectl apply -n {TEST_NAMESPACE} -f -",
+ python=python,
+ kubernetes_version=kubernetes_version,
+ output=output,
+ check=False,
+ shell=True,
+ )
+
+ run_command_with_k8s_env(
+ f"kubectl get serviceaccount -n {HELM_AIRFLOW_NAMESPACE} "
+ "--field-selector metadata.name!=default -o yaml "
+ f"| sed 's/namespace: {HELM_AIRFLOW_NAMESPACE}/namespace:
{TEST_NAMESPACE}/' "
+ f"| kubectl apply -n {TEST_NAMESPACE} -f -",
+ python=python,
+ kubernetes_version=kubernetes_version,
+ output=output,
+ check=False,
+ shell=True,
+ )
+
get_console(output=output).print(
f"\n[success]Airflow for Python {python} and "
f"K8S version {kubernetes_version} has been successfully deployed."
@@ -1073,6 +1123,7 @@ def _deploy_airflow(
@option_debug_resources
@option_include_success_outputs
@option_use_standard_naming
+@option_multi_namespace_mode
@option_python_versions
@option_kubernetes_versions
@option_verbose
@@ -1093,6 +1144,7 @@ def deploy_airflow(
python_versions: str,
kubernetes_versions: str,
extra_options: tuple[str, ...],
+ multi_namespace_mode: bool = False,
):
if run_in_parallel:
python_version_array: list[str] = python_versions.split(" ")
@@ -1121,6 +1173,7 @@ def deploy_airflow(
"wait_time_in_seconds": wait_time_in_seconds,
"extra_options": extra_options,
"output": outputs[index],
+ "multi_namespace_mode": multi_namespace_mode,
},
)
for index, combo in enumerate(combos)
@@ -1142,6 +1195,7 @@ def deploy_airflow(
use_standard_naming=use_standard_naming,
wait_time_in_seconds=wait_time_in_seconds,
extra_options=extra_options,
+ multi_namespace_mode=multi_namespace_mode,
)
if return_code == 0:
get_console().print(
@@ -1486,6 +1540,7 @@ def _run_complete_tests(
use_standard_naming=use_standard_naming,
wait_time_in_seconds=wait_time_in_seconds,
extra_options=extra_options,
+ multi_namespace_mode=True,
)
if returncode != 0:
_logs(python=python, kubernetes_version=kubernetes_version)
@@ -1516,6 +1571,7 @@ def _run_complete_tests(
use_standard_naming=use_standard_naming,
wait_time_in_seconds=wait_time_in_seconds,
extra_options=extra_options,
+ multi_namespace_mode=True,
)
if returncode != 0:
_logs(python=python, kubernetes_version=kubernetes_version)
diff --git
a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py
index 5049100c59..3e988bd6bd 100644
--- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py
@@ -160,6 +160,7 @@ KUBERNETES_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
"--upgrade",
"--wait-time-in-seconds",
"--use-standard-naming",
+ "--multi-namespace-mode",
],
},
{
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index 7c2e6a7230..b7002add2a 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -365,7 +365,7 @@ def create_virtualenv(force_venv_setup: bool) ->
RunCommandResult:
def run_command_with_k8s_env(
- cmd: list[str],
+ cmd: list[str] | str,
python: str,
kubernetes_version: str,
executor: str | None = None,
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py
b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 8cc1f70809..2116ff83ad 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -55,7 +55,7 @@ OPTION_MATCHER = re.compile(r"^[A-Z_]*=.*$")
def run_command(
- cmd: list[str],
+ cmd: list[str] | str,
title: str | None = None,
*,
check: bool = True,
@@ -120,7 +120,7 @@ def run_command(
if not title:
shortened_command = [
shorten_command(index, argument)
- for index, argument in enumerate(cmd)
+ for index, argument in enumerate(cmd if isinstance(cmd, list) else
shlex.split(cmd))
if not exclude_command(index, argument)
]
# Heuristics to get a (possibly) short but explanatory title showing
what the command does
@@ -135,7 +135,7 @@ def run_command(
if "capture_output" not in kwargs or not kwargs["capture_output"]:
kwargs["stdout"] = output.file
kwargs["stderr"] = subprocess.STDOUT
- command_to_print = " ".join(shlex.quote(c) for c in cmd)
+ command_to_print = " ".join(shlex.quote(c) for c in cmd) if
isinstance(cmd, list) else cmd
env_to_print = get_environments_to_print(env)
if not get_verbose(verbose_override) and not get_dry_run(dry_run_override):
return subprocess.run(cmd, input=input, check=check, env=cmd_env,
cwd=workdir, **kwargs)
diff --git a/images/breeze/output_k8s_deploy-airflow.svg
b/images/breeze/output_k8s_deploy-airflow.svg
index c50d3e99ea..7fe972b717 100644
--- a/images/breeze/output_k8s_deploy-airflow.svg
+++ b/images/breeze/output_k8s_deploy-airflow.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 952.8"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 977.1999999999999"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-k8s-deploy-airflow-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="901.8" />
+ <rect x="0" y="0" width="1463.0" height="926.1999999999999" />
</clipPath>
<clipPath id="breeze-k8s-deploy-airflow-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -153,9 +153,12 @@
<clipPath id="breeze-k8s-deploy-airflow-line-35">
<rect x="0" y="855.5" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-k8s-deploy-airflow-line-36">
+ <rect x="0" y="879.9" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="950.8" rx="8"/><text
class="breeze-k8s-deploy-airflow-title" fill="#c5c8c6" text-anchor="middle"
x="740" y="27">Command: k8s deploy-airflow</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="975.2" rx="8"/><text
class="breeze-k8s-deploy-airflow-title" fill="#c5c8c6" text-anchor="middle"
x="740" y="27">Command: k8s deploy-airflow</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -182,26 +185,27 @@
</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="361.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-14)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="361.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-14)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="361.6" textLength="97.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-14)">-upgrade</text><text
class="breeze-k8s-deploy-airflow-r1" x="366" y="361.6" [...]
</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="386"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-15)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="386" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-15)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="386" textLength="61"
clip-path="url(#breeze-k8s-deploy-airflow-line-15)">-wait</text><text
class="breeze-k8s-deploy-airflow-r4" x="97.6" y="386" textLength=" [...]
</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="410.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-16)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="410.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-16)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="410.4" textLength="48.8"
clip-path="url(#breeze-k8s-deploy-airflow-line-16)">-use</text><text
class="breeze-k8s-deploy-airflow-r4" x="85.4" y="410.4" tex [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="434.8"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="434.8" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-17)">
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="459.2"
textLength="24.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-18)">╭─</text><text
class="breeze-k8s-deploy-airflow-r5" x="24.4" y="459.2" textLength="219.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-18)"> Parallel options </text><text
class="breeze-k8s-deploy-airflow-r5" x="244" y="459.2" textLength="1195.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-18)">─────────────────────────────────────────
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="483.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="483.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="483.6" textLength="48.8"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)">-run</text><text
class="breeze-k8s-deploy-airflow-r4" x="85.4" y="483.6" tex [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="508"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="508" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="508" textLength="146.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">-parallelism</text><text
class="breeze-k8s-deploy-airflow-r1" x="378.2" y="508" t [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="532.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-21)">│</text><text
class="breeze-k8s-deploy-airflow-r1" x="378.2" y="532.4" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-21)">operations.                                 
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="556.8"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-22)">│</text><text
class="breeze-k8s-deploy-airflow-r7" x="378.2" y="556.8" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-22)">(INTEGER RANGE)                                &
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="581.2"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-23)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="581.2" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-23)">[default: 2; 1<=x<=4]                             &#
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="605.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-24)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="605.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-24)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="605.6" textLength="85.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-24)">-python</text><text
class="breeze-k8s-deploy-airflow-r4" x="122" y="605.6" t [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="630"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-25)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="630" textLength="951.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-25)">[default: 3.8 3.9 3.10 3.11]                            &#
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="654.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-26)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="654.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-26)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="654.4" textLength="134.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-26)">-kubernetes</text><text
class="breeze-k8s-deploy-airflow-r4" x="170.8" y="6 [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="678.8"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-27)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="678.8" textLength="756.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-27)">[default: v1.25.11 v1.26.6 v1.27.3 v1.28.0]                   </text><text
class="breeze-k8s-deploy [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="703.2"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-28)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="703.2" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-28)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="703.2" textLength="61"
clip-path="url(#breeze-k8s-deploy-airflow-line-28)">-skip</text><text
class="breeze-k8s-deploy-airflow-r4" x="97.6" y="703.2" text [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="727.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="727.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="727.6" textLength="73.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">-debug</text><text
class="breeze-k8s-deploy-airflow-r4" x="109.8" y="727.6" [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="752"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="752" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="752" textLength="97.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">-include</text><text
class="breeze-k8s-deploy-airflow-r4" x="134.2" y="752" textLe [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="776.4"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-31)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="776.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-31)">
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="800.8"
textLength="24.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-32)">╭─</text><text
class="breeze-k8s-deploy-airflow-r5" x="24.4" y="800.8" textLength="195.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-32)"> Common options </text><text
class="breeze-k8s-deploy-airflow-r5" x="219.6" y="800.8" textLength="1220"
clip-path="url(#breeze-k8s-deploy-airflow-line-32)">───────────────────────────────────────────
[...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="825.2"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="825.2" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="825.2" textLength="97.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)">-verbose</text><text
class="breeze-k8s-deploy-airflow-r6" x="158.6" y="825.2 [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="849.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="849.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="849.6" textLength="48.8"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">-dry</text><text
class="breeze-k8s-deploy-airflow-r4" x="85.4" y="849.6" tex [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="874"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="874" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="874" textLength="61"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">-help</text><text
class="breeze-k8s-deploy-airflow-r6" x="158.6" y="874" textLength= [...]
-</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="898.4"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="898.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-36)">
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-17)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="434.8" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-17)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="434.8" textLength="73.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-17)">-multi</text><text
class="breeze-k8s-deploy-airflow-r4" x="109.8" y="434.8" [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="459.2"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="459.2" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-18)">
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="483.6"
textLength="24.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)">╭─</text><text
class="breeze-k8s-deploy-airflow-r5" x="24.4" y="483.6" textLength="219.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)"> Parallel options </text><text
class="breeze-k8s-deploy-airflow-r5" x="244" y="483.6" textLength="1195.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-19)">─────────────────────────────────────────
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="508"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="508" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="508" textLength="48.8"
clip-path="url(#breeze-k8s-deploy-airflow-line-20)">-run</text><text
class="breeze-k8s-deploy-airflow-r4" x="85.4" y="508" textLength= [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="532.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-21)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="532.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-21)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="532.4" textLength="146.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-21)">-parallelism</text><text
class="breeze-k8s-deploy-airflow-r1" x="378.2" y=" [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="556.8"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-22)">│</text><text
class="breeze-k8s-deploy-airflow-r1" x="378.2" y="556.8" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-22)">operations.                                 
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="581.2"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-23)">│</text><text
class="breeze-k8s-deploy-airflow-r7" x="378.2" y="581.2" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-23)">(INTEGER RANGE)                                &
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="605.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-24)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="605.6" textLength="1061.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-24)">[default: 2; 1<=x<=4]                             &#
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="630"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-25)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="630" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-25)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="630" textLength="85.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-25)">-python</text><text
class="breeze-k8s-deploy-airflow-r4" x="122" y="630" textLengt [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="654.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-26)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="654.4" textLength="951.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-26)">[default: 3.8 3.9 3.10 3.11]                           
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="678.8"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-27)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="678.8" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-27)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="678.8" textLength="134.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-27)">-kubernetes</text><text
class="breeze-k8s-deploy-airflow-r4" x="170.8" y="6 [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="703.2"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-28)">│</text><text
class="breeze-k8s-deploy-airflow-r5" x="378.2" y="703.2" textLength="756.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-28)">[default: v1.25.11 v1.26.6 v1.27.3 v1.28.0]                   </text><text
class="breeze-k8s-deploy [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="727.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="727.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="727.6" textLength="61"
clip-path="url(#breeze-k8s-deploy-airflow-line-29)">-skip</text><text
class="breeze-k8s-deploy-airflow-r4" x="97.6" y="727.6" text [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="752"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="752" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="752" textLength="73.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-30)">-debug</text><text
class="breeze-k8s-deploy-airflow-r4" x="109.8" y="752" textLeng [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="776.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-31)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="776.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-31)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="776.4" textLength="97.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-31)">-include</text><text
class="breeze-k8s-deploy-airflow-r4" x="134.2" y="776.4 [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="800.8"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-32)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="800.8" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-32)">
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="825.2"
textLength="24.4"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)">╭─</text><text
class="breeze-k8s-deploy-airflow-r5" x="24.4" y="825.2" textLength="195.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)"> Common options </text><text
class="breeze-k8s-deploy-airflow-r5" x="219.6" y="825.2" textLength="1220"
clip-path="url(#breeze-k8s-deploy-airflow-line-33)">───────────────────────────────────────────
[...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="849.6"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="849.6" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="849.6" textLength="97.6"
clip-path="url(#breeze-k8s-deploy-airflow-line-34)">-verbose</text><text
class="breeze-k8s-deploy-airflow-r6" x="158.6" y="849.6 [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="874"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="874" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="874" textLength="48.8"
clip-path="url(#breeze-k8s-deploy-airflow-line-35)">-dry</text><text
class="breeze-k8s-deploy-airflow-r4" x="85.4" y="874" textLength= [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="898.4"
textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-36)">│</text><text
class="breeze-k8s-deploy-airflow-r4" x="24.4" y="898.4" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-36)">-</text><text
class="breeze-k8s-deploy-airflow-r4" x="36.6" y="898.4" textLength="61"
clip-path="url(#breeze-k8s-deploy-airflow-line-36)">-help</text><text
class="breeze-k8s-deploy-airflow-r6" x="158.6" y="898.4" tex [...]
+</text><text class="breeze-k8s-deploy-airflow-r5" x="0" y="922.8"
textLength="1464"
clip-path="url(#breeze-k8s-deploy-airflow-line-37)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-k8s-deploy-airflow-r1" x="1464" y="922.8" textLength="12.2"
clip-path="url(#breeze-k8s-deploy-airflow-line-37)">
</text>
</g>
</g>
diff --git a/images/breeze/output_k8s_deploy-airflow.txt
b/images/breeze/output_k8s_deploy-airflow.txt
index 17f2b4c07b..b6237009d6 100644
--- a/images/breeze/output_k8s_deploy-airflow.txt
+++ b/images/breeze/output_k8s_deploy-airflow.txt
@@ -1 +1 @@
-f4b05b2101a4a029c9706ecd6fbf3c5c
+e508ceccf1296e7915277e2b8df7263a
diff --git a/tests/providers/cncf/kubernetes/test_pod_generator.py
b/tests/providers/cncf/kubernetes/test_pod_generator.py
index 2739781858..969bf26bc5 100644
--- a/tests/providers/cncf/kubernetes/test_pod_generator.py
+++ b/tests/providers/cncf/kubernetes/test_pod_generator.py
@@ -401,17 +401,29 @@ class TestPodGenerator:
pytest.param(None, "busybox", id="no_image_in_cfg"),
],
)
- def test_construct_pod(self, config_image, expected_image):
+ @pytest.mark.parametrize(
+ "pod_override_object_namespace, expected_namespace",
+ [
+ ("new_namespace", "new_namespace"), # pod_override_object
namespace should be used
+ (None, "test_namespace"), # if it is not provided, we use default
one
+ ],
+ )
+ def test_construct_pod(
+ self, config_image, expected_image, pod_override_object_namespace,
expected_namespace
+ ):
template_file = sys.path[0] +
"/tests/providers/cncf/kubernetes/pod_generator_base_with_secrets.yaml"
worker_config = PodGenerator.deserialize_model_file(template_file)
executor_config = k8s.V1Pod(
+ metadata=k8s.V1ObjectMeta(
+ namespace=pod_override_object_namespace,
+ ),
spec=k8s.V1PodSpec(
containers=[
k8s.V1Container(
name="",
resources=k8s.V1ResourceRequirements(limits={"cpu": "1m", "memory": "1G"})
)
]
- )
+ ),
)
result = PodGenerator.construct_pod(
@@ -432,7 +444,7 @@ class TestPodGenerator:
expected.metadata.labels["app"] = "myapp"
expected.metadata.annotations = self.annotations
expected.metadata.name = "pod_id"
- expected.metadata.namespace = "test_namespace"
+ expected.metadata.namespace = expected_namespace
expected.spec.containers[0].args = ["command"]
expected.spec.containers[0].image = expected_image
expected.spec.containers[0].resources = {"limits": {"cpu": "1m",
"memory": "1G"}}
