Repository: incubator-airflow Updated Branches: refs/heads/master ae6198794 -> 0da540bf7
[AIRFLOW-1145] Fix closest_date_partition function with before set to True If we're looking for the closest date before, we should take the latest date in the list of date before. Closes #2257 from julien-gm/fix_closest-date- partition Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/0da540bf Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/0da540bf Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/0da540bf Branch: refs/heads/master Commit: 0da540bf7840ae3cac866e352fba2b8b5cd9a625 Parents: ae61987 Author: julien-gm <[email protected]> Authored: Sat May 13 21:10:13 2017 +0200 Committer: Bolke de Bruin <[email protected]> Committed: Sat May 13 21:10:13 2017 +0200 ---------------------------------------------------------------------- airflow/macros/hive.py | 4 ++-- tests/macros/__init__.py | 14 ++++++++++++++ tests/macros/test_hive.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0da540bf/airflow/macros/hive.py ---------------------------------------------------------------------- diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py index 13b8234..c68c293 100644 --- a/airflow/macros/hive.py +++ b/airflow/macros/hive.py @@ -61,8 +61,8 @@ def _closest_date(target_dt, date_list, before_target=None): :returns: The closest date :rtype: datetime.date or None ''' - fb = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max - fa = lambda d: d - target_dt if d <= target_dt else datetime.timedelta.min + fb = lambda d: target_dt - d if d <= target_dt else datetime.timedelta.max + fa = lambda d: d - target_dt if d >= target_dt else datetime.timedelta.max fnone = lambda d: target_dt - d if d < target_dt else d - target_dt if before_target is None: return min(date_list, key=fnone).date() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0da540bf/tests/macros/__init__.py ---------------------------------------------------------------------- diff --git a/tests/macros/__init__.py b/tests/macros/__init__.py new file mode 100644 index 0000000..759b563 --- /dev/null +++ b/tests/macros/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0da540bf/tests/macros/test_hive.py ---------------------------------------------------------------------- diff --git a/tests/macros/test_hive.py b/tests/macros/test_hive.py new file mode 100644 index 0000000..17ee68f --- /dev/null +++ b/tests/macros/test_hive.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta +import unittest + +from airflow.macros import hive + + +class Hive(unittest.TestCase): + def test_closest_ds_partition(self): + d1 = datetime.strptime('2017-04-24', '%Y-%m-%d') + d2 = datetime.strptime('2017-04-25', '%Y-%m-%d') + d3 = datetime.strptime('2017-04-26', '%Y-%m-%d') + d4 = datetime.strptime('2017-04-28', '%Y-%m-%d') + d5 = datetime.strptime('2017-04-29', '%Y-%m-%d') + target_dt = datetime.strptime('2017-04-27', '%Y-%m-%d') + date_list = [d1, d2, d3, d4, d5] + + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list, True))) + self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, date_list, False))) + + # when before is not set, the closest date should be returned + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, [d1, d2, d3, d5], None))) + self.assertEquals("2017-04-28", str(hive._closest_date(target_dt, [d1, d2, d4, d5]))) + self.assertEquals("2017-04-26", str(hive._closest_date(target_dt, date_list))) + + +if __name__ == '__main__': + unittest.main()
