Repository: incubator-airflow Updated Branches: refs/heads/v1-8-test 0db3249df -> 9e6dfcbdb
[AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before. Closes #2257 from julien-gm/fix_closest-date- partition (cherry picked from commit 0da540bf7840ae3cac866e352fba2b8b5cd9a625) Signed-off-by: Bolke de Bruin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/9e6dfcbd Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/9e6dfcbd Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/9e6dfcbd Branch: refs/heads/v1-8-test Commit: 9e6dfcbdb4a4bbd63a8683778e3b3dafe149150c Parents: 0db3249 Author: julien-gm <[email protected]> Authored: Sat May 13 21:10:13 2017 +0200 Committer: Bolke de Bruin <[email protected]> Committed: Sat May 13 21:10:46 2017 +0200 ---------------------------------------------------------------------- airflow/macros/hive.py | 4 ++-- tests/macros/__init__.py | 14 ++++++++++++++ tests/macros/test_hive.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/airflow/macros/hive.py ---------------------------------------------------------------------- diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py index 13b8234..c68c293 100644 --- a/airflow/macros/hive.py +++ b/airflow/macros/hive.py @@ -61,8 +61,8 @@ def _closest_date(target_dt, date_list, before_target=None): :returns: The closest date :rtype: datetime.date or None ''' - fb = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max - fa = lambda d: d - target_dt if d <= target_dt else datetime.timedelta.min + fb = lambda d: target_dt - d if d <= target_dt else datetime.timedelta.max + fa = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max fnone = lambda d: target_dt - d if d < target_dt else d - target_dt if before_target is None: return min(date_list, key=fnone).date() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/__init__.py ---------------------------------------------------------------------- diff --git a/tests/macros/__init__.py b/tests/macros/__init__.py new file mode 100644 index 0000000..759b563 --- /dev/null +++ b/tests/macros/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/9e6dfcbd/tests/macros/test_hive.py ---------------------------------------------------------------------- diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py new file mode 100644 index 0000000..17ee68f --- /dev/null +++ b/tests/macros/test_hive.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta +import unittest + +from airflow.macros import hive + + +class Hive(unittest.TestCase): + def test_closest_ds_partition(self): + d1 = datetime.strptime('2017-04-24', '%Y-%m-%d') + d2 = datetime.strptime('2017-04-25', '%Y-%m-%d') + d3 = datetime.strptime('2017-04-26', '%Y-%m-%d') + d4 = datetime.strptime('2017-04-28', '%Y-%m-%d') + d5 = datetime.strptime('2017-04-29', '%Y-%m-%d') + target_dt = datetime.strptime('2017-04-27', '%Y-%m-%d') + date_list = [d1, d2, d3, d4, d5] + + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list, True))) + self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, date_list, False))) + + # when before is not set, the closest date should be returned + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5], None))) + self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5]))) + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list))) + + +if __name__ == '__main__': + unittest.main()
