This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 8f9538e Add Docs for ``default_pool`` slots (#15997)
8f9538e is described below
commit 8f9538e4cc52c2bade6b33f25b04f09f0c8ec6ce
Author: Kaxil Naik <[email protected]>
AuthorDate: Sat May 22 12:56:22 2021 +0100
Add Docs for ``default_pool`` slots (#15997)
Until now this was undocumented, however it was used in:
https://github.com/apache/airflow/blob/aa4713e43f92d3e4c68c3ad00e2d44caaf29aafe/airflow/utils/db.py#L75
This was done in 1.10.4 in
https://github.com/apache/airflow/commit/2c99ec624bd66e9fa38e9f0087d46ef4d7f05aec
commit. This commit also updates the name from `non_pooled_task_slot_count`
to `default_pool_task_slot_count` as
this is what it actually does.
---
UPDATING.md | 10 ++++++++++
airflow/config_templates/config.yml | 9 +++++++++
airflow/config_templates/default_airflow.cfg | 5 +++++
airflow/configuration.py | 1 +
airflow/utils/db.py | 2 +-
5 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/UPDATING.md b/UPDATING.md
index bcf4d4f..2bf424c 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -72,6 +72,16 @@ https://developers.google.com/style/inclusive-documentation
-->
+### Default Task Pools Slots can be set using ``[core]
default_pool_task_slot_count``
+
+By default tasks are running in `default_pool`. `default_pool` is initialized
with `128` slots and user can change the
+number of slots through UI/CLI/API for an existing deployment.
+
+For new deployments, you can use `default_pool_task_slot_count` setting in
`[core]` section. This setting would
+not have any effect in an existing deployment where the ``default_pool``
already exists.
+
+Previously this was controlled by `non_pooled_task_slot_count` in `[core]`
section, which was not documented.
+
## Airflow 2.1.0
### New "deprecated_api" extra
diff --git a/airflow/config_templates/config.yml
b/airflow/config_templates/config.yml
index 39d2539..158bf18 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -435,6 +435,15 @@
type: string
example: ~
default: ""
+ - name: default_pool_task_slot_count
+ description: |
+ Task Slot counts for ``default_pool``. This setting would not have any
effect in an existing
+ deployment where the ``default_pool`` is already created. For existing
deployments, users can
+ change the number of slots using Webserver, API or the CLI
+ version_added: 2.2.0
+ type: string
+ example: ~
+ default: "128"
- name: logging
description: ~
diff --git a/airflow/config_templates/default_airflow.cfg
b/airflow/config_templates/default_airflow.cfg
index bf033ef..6b4d734 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -250,6 +250,11 @@ hide_sensitive_var_conn_fields = True
# extra JSON.
sensitive_var_conn_names =
+# Task Slot counts for ``default_pool``. This setting would not have any
effect in an existing
+# deployment where the ``default_pool`` is already created. For existing
deployments, users can
+# change the number of slots using Webserver, API or the CLI
+default_pool_task_slot_count = 128
+
[logging]
# The folder where airflow should store its log files
# This path must be absolute
diff --git a/airflow/configuration.py b/airflow/configuration.py
index d60c2e1..1635700 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -162,6 +162,7 @@ class AirflowConfigParser(ConfigParser): # pylint:
disable=too-many-ancestors
('operators', 'default_queue'): ('celery', 'default_queue', '2.1.0'),
('core', 'hide_sensitive_var_conn_fields'): ('admin',
'hide_sensitive_variable_fields', '2.1.0'),
('core', 'sensitive_var_conn_names'): ('admin',
'sensitive_variable_fields', '2.1.0'),
+ ('core', 'default_pool_task_slot_count'): ('core',
'non_pooled_task_slot_count', '1.10.4'),
}
# A mapping of old default values that we want to change and warn the user
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 979e020..50edaec 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -72,7 +72,7 @@ def add_default_pool_if_not_exists(session=None):
if not Pool.get_pool(Pool.DEFAULT_POOL_NAME, session=session):
default_pool = Pool(
pool=Pool.DEFAULT_POOL_NAME,
- slots=conf.getint(section='core',
key='non_pooled_task_slot_count', fallback=128),
+ slots=conf.getint(section='core',
key='default_pool_task_slot_count'),
description="Default pool",
)
session.add(default_pool)
