Repository: incubator-airflow Updated Branches: refs/heads/master 71594751c -> f5ffb9deb
[AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an array.[ Closes #2382 from ashb/spark-submit-operator- preserve-spaces Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/f5ffb9de Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/f5ffb9de Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/f5ffb9de Branch: refs/heads/master Commit: f5ffb9deb51e896cdda8992c8b28102f7d650662 Parents: 7159475 Author: Ash Berlin-Taylor <[email protected]> Authored: Wed Jul 12 11:27:39 2017 +0200 Committer: Bolke de Bruin <[email protected]> Committed: Wed Jul 12 11:27:39 2017 +0200 ---------------------------------------------------------------------- airflow/contrib/hooks/spark_submit_hook.py | 8 ++--- tests/contrib/hooks/test_spark_submit_hook.py | 6 ++-- .../operators/test_spark_submit_operator.py | 31 +++++++++++--------- 3 files changed, 23 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/airflow/contrib/hooks/spark_submit_hook.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/spark_submit_hook.py b/airflow/contrib/hooks/spark_submit_hook.py index f3d6e34..88d547b 100644 --- a/airflow/contrib/hooks/spark_submit_hook.py +++ b/airflow/contrib/hooks/spark_submit_hook.py @@ -194,12 +194,8 @@ class SparkSubmitHook(BaseHook): # Append any application arguments if self._application_args: - for arg in self._application_args: - if len(arg.split()) > 1: - for splitted_option in arg.split(): - connection_cmd += [splitted_option] - else: - connection_cmd += [arg] + connection_cmd += self._application_args + logging.debug("Spark-Submit cmd: {}".format(connection_cmd)) return connection_cmd http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/hooks/test_spark_submit_hook.py ---------------------------------------------------------------------- diff --git a/tests/contrib/hooks/test_spark_submit_hook.py b/tests/contrib/hooks/test_spark_submit_hook.py index ef0857b..98e959b 100644 --- a/tests/contrib/hooks/test_spark_submit_hook.py +++ b/tests/contrib/hooks/test_spark_submit_hook.py @@ -45,8 +45,9 @@ class TestSparkSubmitHook(unittest.TestCase): 'driver_memory': '3g', 'java_class': 'com.foo.bar.AppMain', 'application_args': [ - '-f foo', - '--bar bar', + '-f', 'foo', + '--bar', 'bar', + '--with-spaces', 'args should keep embdedded spaces', 'baz' ] } @@ -129,6 +130,7 @@ class TestSparkSubmitHook(unittest.TestCase): 'test_application.py', '-f', 'foo', '--bar', 'bar', + '--with-spaces', 'args should keep embdedded spaces', 'baz' ] self.assertEquals(expected_build_cmd, cmd) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/operators/test_spark_submit_operator.py ---------------------------------------------------------------------- diff --git a/tests/contrib/operators/test_spark_submit_operator.py b/tests/contrib/operators/test_spark_submit_operator.py index 09c5a93..1d41941 100644 --- a/tests/contrib/operators/test_spark_submit_operator.py +++ b/tests/contrib/operators/test_spark_submit_operator.py @@ -46,10 +46,11 @@ class TestSparkSubmitOperator(unittest.TestCase): 'driver_memory': '3g', 'java_class': 'com.foo.bar.AppMain', 'application_args': [ - '-f foo', - '--bar bar', - '--start {{ macros.ds_add(ds, -1)}}', - '--end {{ ds }}' + '-f', 'foo', + '--bar', 'bar', + '--start', '{{ macros.ds_add(ds, -1)}}', + '--end', '{{ ds }}', + '--with-spaces', 'args should keep embdedded spaces', ] } @@ -95,10 +96,11 @@ class TestSparkSubmitOperator(unittest.TestCase): 'driver_memory': '3g', 'java_class': 'com.foo.bar.AppMain', 'application_args': [ - '-f foo', - '--bar bar', - '--start {{ macros.ds_add(ds, -1)}}', - '--end {{ ds }}' + '-f', 'foo', + '--bar', 'bar', + '--start', '{{ macros.ds_add(ds, -1)}}', + '--end', '{{ ds }}', + '--with-spaces', 'args should keep embdedded spaces', ] } @@ -130,14 +132,15 @@ class TestSparkSubmitOperator(unittest.TestCase): ti.render_templates() # Then - expected_application_args = [u'-f foo', - u'--bar bar', - u'--start %s' % (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"), - u'--end %s' % DEFAULT_DATE.strftime("%Y-%m-%d")] + expected_application_args = [u'-f', 'foo', + u'--bar', 'bar', + u'--start', (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"), + u'--end', DEFAULT_DATE.strftime("%Y-%m-%d"), + u'--with-spaces', u'args should keep embdedded spaces', + ] expected_name = "spark_submit_job" - self.assertListEqual(sorted(expected_application_args), sorted(getattr(operator, '_application_args'))) + self.assertListEqual(expected_application_args, getattr(operator, '_application_args')) self.assertEqual(expected_name, getattr(operator, '_name')) - if __name__ == '__main__': unittest.main()
