Repository: incubator-airflow Updated Branches: refs/heads/v1-9-test bd377fb2e -> 48a95ef92
[AIRFLOW-1601] Add configurable task cleanup time When task processes are SIGTERMed, they have by default 5 seconds to cleanup before a SIGKILL arrives. This allows this value to be configurable. Closes #2601 from saguziel/aguziel-configure-task- killer (cherry picked from commit 17189ddcc8f7b8efaca6384ea854f5573e11ea4e) Signed-off-by: Alex Guziel <alex.guz...@airbnb.com> Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/48a95ef9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/48a95ef9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/48a95ef9 Branch: refs/heads/v1-9-test Commit: 48a95ef9299dd295573edb5220c395ab1000450c Parents: bd377fb Author: Alex Guziel <alex.guz...@airbnb.com> Authored: Wed Sep 13 14:22:48 2017 -0700 Committer: Alex Guziel <alex.guz...@airbnb.com> Committed: Wed Sep 13 14:23:12 2017 -0700 ---------------------------------------------------------------------- airflow/config_templates/default_airflow.cfg | 4 ++++ airflow/config_templates/default_test.cfg | 1 + airflow/utils/helpers.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/48a95ef9/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 948c72c..94efe60 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -133,6 +133,10 @@ task_log_reader = file.task # RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). enable_xcom_pickling = True +# When a task is killed forcefully, this is the amount of time in seconds that +# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED +killed_task_cleanup_time = 60 + [cli] # In what way should the cli access the API. The LocalClient will use the # database directly, while the json_client will use the api running on the http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/48a95ef9/airflow/config_templates/default_test.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg index 2a090d4..b065313 100644 --- a/airflow/config_templates/default_test.cfg +++ b/airflow/config_templates/default_test.cfg @@ -38,6 +38,7 @@ dags_are_paused_at_creation = False fernet_key = {FERNET_KEY} non_pooled_task_slot_count = 128 enable_xcom_pickling = False +killed_task_cleanup_time = 5 [cli] api_client = airflow.api.client.local_client http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/48a95ef9/airflow/utils/helpers.py ---------------------------------------------------------------------- diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 9a94125..4b8944e 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -31,11 +31,12 @@ import subprocess import sys import warnings +from airflow import configuration from airflow.exceptions import AirflowException # When killing processes, time to wait after issuing a SIGTERM before issuing a # SIGKILL. -DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM = 5 +DEFAULT_TIME_TO_WAIT_AFTER_SIGTERM = configuration.getint('core', 'KILLED_TASK_CLEANUP_TIME') def validate_key(k, max_length=250):
