Repository: incubator-airflow Updated Branches: refs/heads/master 4386cd414 -> 263861ed2
[AIRFLOW-1657] Handle failing qubole operator Closes #2643 from rupesh92/AIRFLOW-1657 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/263861ed Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/263861ed Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/263861ed Branch: refs/heads/master Commit: 263861ed2ccefc21665697af4903bdb021431dea Parents: 4386cd4 Author: Rupesh Bansal <[email protected]> Authored: Mon Oct 23 14:48:30 2017 +0530 Committer: Sumit Maheshwari <[email protected]> Committed: Mon Oct 23 14:48:30 2017 +0530 ---------------------------------------------------------------------- airflow/contrib/hooks/qubole_hook.py | 5 +---- tests/contrib/operators/test_qubole_operator.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/263861ed/airflow/contrib/hooks/qubole_hook.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/qubole_hook.py b/airflow/contrib/hooks/qubole_hook.py index 833c1c7..f3bcc20 100755 --- a/airflow/contrib/hooks/qubole_hook.py +++ b/airflow/contrib/hooks/qubole_hook.py @@ -202,9 +202,6 @@ class QuboleHook(BaseHook, LoggingMixin): args.append("--tags={0}".format(','.join(filter(None,tags)))) if inplace_args is not None: - if cmd_type == 'hadoopcmd': - args += inplace_args.split(' ', 1) - else: - args += inplace_args.split(' ') + args += inplace_args.split(' ') return args http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/263861ed/tests/contrib/operators/test_qubole_operator.py ---------------------------------------------------------------------- diff --git a/tests/contrib/operators/test_qubole_operator.py b/tests/contrib/operators/test_qubole_operator.py index 0e6e13d..9b5bf3a 100644 --- a/tests/contrib/operators/test_qubole_operator.py +++ b/tests/contrib/operators/test_qubole_operator.py @@ -89,6 +89,19 @@ class QuboleOperatorTest(unittest.TestCase): self.assertEqual(task.get_hook().create_cmd_args({'run_id':'dummy'})[2], "key2=value2") - + task = QuboleOperator(task_id=TASK_ID, command_type='hadoopcmd', + sub_command="s3distcp --src s3n://airflow/source_hadoopcmd " + + "--dest s3n://airflow/destination_hadoopcmd", dag=dag) + + self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[1], + "s3distcp") + self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[2], + "--src") + self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[3], + "s3n://airflow/source_hadoopcmd") + self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[4], + "--dest") + self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[5], + "s3n://airflow/destination_hadoopcmd")
