Repository: incubator-airflow Updated Branches: refs/heads/master 0e892ccd7 -> 7945854cc
[AIRFLOW-2532] Support logs_volume_subpath for KubernetesExecutor The kubernetes section in the configuration file supports logs_volume_subpath for KubernetesExecutor. Closes #3430 from imroc/AIRFLOW-2532 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/7945854c Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/7945854c Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/7945854c Branch: refs/heads/master Commit: 7945854cc4885c2314dc8b7a73c87422823bea86 Parents: 0e892cc Author: roc <[email protected]> Authored: Wed May 30 11:14:13 2018 +0200 Committer: Fokko Driesprong <[email protected]> Committed: Wed May 30 11:14:13 2018 +0200 ---------------------------------------------------------------------- airflow/config_templates/default_airflow.cfg | 3 +++ airflow/contrib/executors/kubernetes_executor.py | 5 +++++ airflow/contrib/kubernetes/worker_configuration.py | 3 ++- scripts/ci/kubernetes/kube/configmaps.yaml | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7945854c/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index ebeb2e6..9c7c96e 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -559,6 +559,9 @@ dags_volume_subpath = # For DAGs mounted via a volume claim (mutually exclusive with volume claim) dags_volume_claim = +# For volume mounted logs, the worker will look in this subpath for logs +logs_volume_subpath = + # A shared volume claim for the logs logs_volume_claim = http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7945854c/airflow/contrib/executors/kubernetes_executor.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/executors/kubernetes_executor.py b/airflow/contrib/executors/kubernetes_executor.py index dff0099..74e4ae5 100644 --- a/airflow/contrib/executors/kubernetes_executor.py +++ b/airflow/contrib/executors/kubernetes_executor.py @@ -133,6 +133,11 @@ class KubeConfig: self.dags_volume_subpath = conf.get( self.kubernetes_section, 'dags_volume_subpath') + # This prop may optionally be set for PV Claims and is used to locate logs + # on a SubPath + self.logs_volume_subpath = conf.get( + self.kubernetes_section, 'logs_volume_subpath') + # This prop may optionally be set for PV Claims and is used to write logs self.base_log_folder = configuration.get(self.core_section, 'base_log_folder') http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7945854c/airflow/contrib/kubernetes/worker_configuration.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/kubernetes/worker_configuration.py b/airflow/contrib/kubernetes/worker_configuration.py index 89f0902..ea0aa23 100644 --- a/airflow/contrib/kubernetes/worker_configuration.py +++ b/airflow/contrib/kubernetes/worker_configuration.py @@ -128,7 +128,8 @@ class WorkerConfiguration(LoggingMixin): ), _construct_volume( logs_volume_name, - self.kube_config.logs_volume_claim + self.kube_config.logs_volume_claim, + self.kube_config.logs_volume_subpath ) ] volume_mounts = [{ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7945854c/scripts/ci/kubernetes/kube/configmaps.yaml ---------------------------------------------------------------------- diff --git a/scripts/ci/kubernetes/kube/configmaps.yaml b/scripts/ci/kubernetes/kube/configmaps.yaml index ddba098..c341f1a 100644 --- a/scripts/ci/kubernetes/kube/configmaps.yaml +++ b/scripts/ci/kubernetes/kube/configmaps.yaml @@ -185,7 +185,9 @@ data: git_user = git_password = dags_volume_claim = airflow-dags + dags_volume_subpath = logs_volume_claim = airflow-logs + logs_volume_subpath = in_cluster = True namespace = default gcp_service_account_keys =
