feng-tao closed pull request #4340: [AIRFLOW-3540] Respect environment config when looking up config file. URL: https://github.com/apache/incubator-airflow/pull/4340
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/UPDATING.md b/UPDATING.md index 986d3a23c1..575fc0a3c5 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,14 @@ assists users migrating to a new version. ## Airflow Master +### Modification to config file discovery + +If the `AIRFLOW_CONFIG` environment variable was not set and the +`~/airflow/airflow.cfg` file existed, airflow previously used +`~/airflow/airflow.cfg` instead of `$AIRFLOW_HOME/airflow.cfg`. Now airflow +will discover its config file using the `$AIRFLOW_CONFIG` and `$AIRFLOW_HOME` +environment variables rather than checking for the presence of a file. + ### Modification to `ts_nodash` macro `ts_nodash` previously contained TimeZone information alongwith execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes. diff --git a/airflow/configuration.py b/airflow/configuration.py index 3662df8d06..332c069a3a 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -441,23 +441,23 @@ def mkdir_p(path): 'Error creating {}: {}'.format(path, exc.strerror)) -# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using -# "~/airflow" and "~/airflow/airflow.cfg" respectively as defaults. +def get_airflow_home(): + return expand_env_var(os.environ.get('AIRFLOW_HOME', '~/airflow')) + + +def get_airflow_config(airflow_home): + if 'AIRFLOW_CONFIG' not in os.environ: + return os.path.join(airflow_home, 'airflow.cfg') + return expand_env_var(os.environ['AIRFLOW_CONFIG']) -if 'AIRFLOW_HOME' not in os.environ: - AIRFLOW_HOME = expand_env_var('~/airflow') -else: - AIRFLOW_HOME = expand_env_var(os.environ['AIRFLOW_HOME']) +# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using +# "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults. + +AIRFLOW_HOME = get_airflow_home() +AIRFLOW_CONFIG = get_airflow_config(AIRFLOW_HOME) mkdir_p(AIRFLOW_HOME) -if 'AIRFLOW_CONFIG' not in os.environ: - if os.path.isfile(expand_env_var('~/airflow.cfg')): - AIRFLOW_CONFIG = expand_env_var('~/airflow.cfg') - else: - AIRFLOW_CONFIG = AIRFLOW_HOME + '/airflow.cfg' -else: - AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG']) # Set up dags folder for unit tests # this directory won't exist if users install via pip diff --git a/tests/test_configuration.py b/tests/test_configuration.py index acebd5732c..9f903f58b3 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -21,6 +21,7 @@ from __future__ import unicode_literals import os +import contextlib from collections import OrderedDict import six @@ -35,6 +36,23 @@ import unittest +@contextlib.contextmanager +def env_vars(**vars): + original = {} + for key, value in vars.items(): + original[key] = os.environ.get(key) + if value is not None: + os.environ[key] = value + else: + os.environ.pop(key, None) + yield + for key, value in original.items(): + if value is not None: + os.environ[key] = value + else: + os.environ.pop(key, None) + + class ConfTest(unittest.TestCase): @classmethod @@ -49,6 +67,30 @@ def tearDownClass(cls): del os.environ['AIRFLOW__TESTSECTION__TESTKEY'] del os.environ['AIRFLOW__TESTSECTION__TESTPERCENT'] + def test_airflow_home_default(self): + with env_vars(AIRFLOW_HOME=None): + self.assertEqual( + configuration.get_airflow_home(), + configuration.expand_env_var('~/airflow')) + + def test_airflow_home_override(self): + with env_vars(AIRFLOW_HOME='/path/to/airflow'): + self.assertEqual( + configuration.get_airflow_home(), + '/path/to/airflow') + + def test_airflow_config_default(self): + with env_vars(AIRFLOW_CONFIG=None): + self.assertEqual( + configuration.get_airflow_config('/home/airflow'), + configuration.expand_env_var('/home/airflow/airflow.cfg')) + + def test_airflow_config_override(self): + with env_vars(AIRFLOW_CONFIG='/path/to/airflow/airflow.cfg'): + self.assertEqual( + configuration.get_airflow_config('/home//airflow'), + '/path/to/airflow/airflow.cfg') + def test_env_var_config(self): opt = conf.get('testsection', 'testkey') self.assertEqual(opt, 'testvalue') ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services