Repository: incubator-airflow Updated Branches: refs/heads/master 6b3f6cee1 -> c5fa8cd41
[AIRFLOW-2411] add dataproc_jars to templated_fields This commit makes it possible to use jinja templates when passing JAR file URIs to the DataProc operators that require JAR files, specifically the DataProc Hive, Pig, SparkSql, Spark, Hadoop and PySpark operators. Closes #3305 from mchalek/template-dataproc-jars Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/c5fa8cd4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/c5fa8cd4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/c5fa8cd4 Branch: refs/heads/master Commit: c5fa8cd411ca67889eafd109988d2472ccfbba10 Parents: 6b3f6ce Author: Kevin McHale <[email protected]> Authored: Thu May 3 22:55:10 2018 +0200 Committer: Fokko Driesprong <[email protected]> Committed: Thu May 3 22:55:10 2018 +0200 ---------------------------------------------------------------------- airflow/contrib/operators/dataproc_operator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/c5fa8cd4/airflow/contrib/operators/dataproc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index 56ebb91..ad0aa09 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -481,7 +481,7 @@ class DataProcPigOperator(BaseOperator): :param region: The specified region where the dataproc cluster is created. :type region: string """ - template_fields = ['query', 'variables', 'job_name', 'cluster_name'] + template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars'] template_ext = ('.pg', '.pig',) ui_color = '#0273d4' @@ -561,7 +561,7 @@ class DataProcHiveOperator(BaseOperator): :param region: The specified region where the dataproc cluster is created. :type region: string """ - template_fields = ['query', 'variables', 'job_name', 'cluster_name'] + template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars'] template_ext = ('.q',) ui_color = '#0273d4' @@ -642,7 +642,7 @@ class DataProcSparkSqlOperator(BaseOperator): :param region: The specified region where the dataproc cluster is created. :type region: string """ - template_fields = ['query', 'variables', 'job_name', 'cluster_name'] + template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars'] template_ext = ('.q',) ui_color = '#0273d4' @@ -731,7 +731,7 @@ class DataProcSparkOperator(BaseOperator): :type region: string """ - template_fields = ['arguments', 'job_name', 'cluster_name'] + template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars'] ui_color = '#0273d4' @apply_defaults @@ -821,7 +821,7 @@ class DataProcHadoopOperator(BaseOperator): :type region: string """ - template_fields = ['arguments', 'job_name', 'cluster_name'] + template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars'] ui_color = '#0273d4' @apply_defaults @@ -911,7 +911,7 @@ class DataProcPySparkOperator(BaseOperator): :type region: string """ - template_fields = ['arguments', 'job_name', 'cluster_name'] + template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars'] ui_color = '#0273d4' @staticmethod
