Repository: incubator-airflow Updated Branches: refs/heads/v1-8-test 5782ad4b6 -> df9464bba
[AIRFLOW-737] Fix HDFS Sensor directory. Due to a bad ordering in the fake snakebite client, one test was wrongly True. Closes #1989 from vfoucault/fixbug/hdfssensor_folder Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8fd65f8a Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8fd65f8a Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8fd65f8a Branch: refs/heads/v1-8-test Commit: 8fd65f8ac4a305da01324c5120fbfab8b51f5d46 Parents: 4a5e17b Author: Vianney Foucault <vianney.fouca...@gmail.com> Authored: Fri Jan 13 21:15:14 2017 +0100 Committer: Bolke de Bruin <bo...@xs4all.nl> Committed: Fri Jan 13 21:15:14 2017 +0100 ---------------------------------------------------------------------- airflow/contrib/sensors/hdfs_sensors.py | 1 + tests/core.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/airflow/contrib/sensors/hdfs_sensors.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/sensors/hdfs_sensors.py b/airflow/contrib/sensors/hdfs_sensors.py index a12e295..4e9bb9b 100644 --- a/airflow/contrib/sensors/hdfs_sensors.py +++ b/airflow/contrib/sensors/hdfs_sensors.py @@ -62,6 +62,7 @@ class HdfsSensorFolder(HdfsSensor): return len(result) == 1 and result[0]['path'] == self.filepath else: logging.info('Poking for filepath {self.filepath} to a non empty directory'.format(**locals())) + result.pop(0) return bool(result) and result[0]['file_type'] == 'f' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8fd65f8a/tests/core.py ---------------------------------------------------------------------- diff --git a/tests/core.py b/tests/core.py index 6fb735f..0f7e41d 100644 --- a/tests/core.py +++ b/tests/core.py @@ -1840,10 +1840,6 @@ class FakeSnakeBiteClient(object): {'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time': 0, 'block_replication': 0, 'modification_time': 1481132141540, 'length': 0, 'blocksize': 0, 'owner': u'hdfs', 'path': '/datadirectory/empty_directory'}] - elif path[0] == '/datadirectory/not_empty_directory': - return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796, - 'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728, - 'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}] elif path[0] == '/datadirectory/not_empty_directory' and include_toplevel: return [ {'group': u'supergroup', 'permission': 493, 'file_type': 'd', 'access_time': 0, 'block_replication': 0, @@ -1852,6 +1848,10 @@ class FakeSnakeBiteClient(object): {'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796, 'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728, 'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}] + elif path[0] == '/datadirectory/not_empty_directory': + return [{'group': u'supergroup', 'permission': 420, 'file_type': 'f', 'access_time': 1481122343796, + 'block_replication': 3, 'modification_time': 1481122343862, 'length': 0, 'blocksize': 134217728, + 'owner': u'hdfs', 'path': '/datadirectory/not_empty_directory/test_file'}] elif path[0] == '/datadirectory/not_existing_file_or_directory': raise FakeSnakeBiteClientException elif path[0] == '/datadirectory/regex_dir':
