Repository: incubator-airflow Updated Branches: refs/heads/master 7af20fe45 -> 8e23d2870
[AIRFLOW-1289] Removes restriction on number of scheduler threads This removes the restriction that the number of threads can be at most the number of CPU cores. There's no reason to have this restriction. Closes #2353 from saguziel/aguziel-increase-cores Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8e23d287 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8e23d287 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8e23d287 Branch: refs/heads/master Commit: 8e23d28705e1f0d9c4ab76bb2318025bab3aef0a Parents: 7af20fe Author: Alex Guziel <[email protected]> Authored: Thu Jun 8 17:27:19 2017 -0700 Committer: Alex Guziel <[email protected]> Committed: Thu Jun 8 17:27:19 2017 -0700 ---------------------------------------------------------------------- airflow/config_templates/default_airflow.cfg | 3 +-- airflow/jobs.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8e23d287/airflow/config_templates/default_airflow.cfg ---------------------------------------------------------------------- diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 2b5fb5d..c6c1da2 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -326,8 +326,7 @@ statsd_port = 8125 statsd_prefix = airflow # The scheduler can run multiple threads in parallel to schedule dags. -# This defines how many threads will run. However airflow will never -# use more threads than the amount of cpu cores available. +# This defines how many threads will run. max_threads = 2 authenticate = False http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8e23d287/airflow/jobs.py ---------------------------------------------------------------------- diff --git a/airflow/jobs.py b/airflow/jobs.py index 508564a..57688d3 100644 --- a/airflow/jobs.py +++ b/airflow/jobs.py @@ -528,7 +528,7 @@ class SchedulerJob(BaseJob): super(SchedulerJob, self).__init__(*args, **kwargs) self.heartrate = conf.getint('scheduler', 'SCHEDULER_HEARTBEAT_SEC') - self.max_threads = min(conf.getint('scheduler', 'max_threads'), multiprocessing.cpu_count()) + self.max_threads = conf.getint('scheduler', 'max_threads') self.using_sqlite = False if 'sqlite' in conf.get('core', 'sql_alchemy_conn'): if self.max_threads > 1:
