[
https://issues.apache.org/jira/browse/AIRFLOW-3245?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16685204#comment-16685204
]
ASF GitHub Bot commented on AIRFLOW-3245:
-----------------------------------------
Fokko closed pull request #4086: [AIRFLOW-3245] fix list processing in
resolve_template_files
URL: https://github.com/apache/incubator-airflow/pull/4086
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/models.py b/airflow/models.py
index fa33609852..35ece3617f 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -2902,14 +2902,24 @@ def resolve_template_files(self):
# Getting the content of files for template_field / template_ext
for attr in self.template_fields:
content = getattr(self, attr)
- if content is not None and \
- isinstance(content, six.string_types) and \
+ if content is None:
+ continue
+ elif isinstance(content, six.string_types) and \
any([content.endswith(ext) for ext in self.template_ext]):
env = self.dag.get_template_env()
try:
setattr(self, attr, env.loader.get_source(env, content)[0])
except Exception as e:
self.log.exception(e)
+ elif isinstance(content, list):
+ env = self.dag.get_template_env()
+ for i in range(len(content)):
+ if isinstance(content[i], six.string_types) and \
+ any([content[i].endswith(ext) for ext in
self.template_ext]):
+ try:
+ content[i] = env.loader.get_source(env,
content[i])[0]
+ except Exception as e:
+ self.log.exception(e)
self.prepare_template()
@property
diff --git a/tests/models.py b/tests/models.py
index b16055b380..9f439605e1 100644
--- a/tests/models.py
+++ b/tests/models.py
@@ -454,6 +454,51 @@ def jinja_udf(name):
result = task.render_template('', "{{ 'world' | hello}}", dict())
self.assertEqual(result, 'Hello world')
+ def test_resolve_template_files_value(self):
+
+ with NamedTemporaryFile(suffix='.template') as f:
+ f.write('{{ ds }}'.encode('utf8'))
+ f.flush()
+ template_dir = os.path.dirname(f.name)
+ template_file = os.path.basename(f.name)
+
+ dag = DAG('test-dag',
+ start_date=DEFAULT_DATE,
+ template_searchpath=template_dir)
+
+ with dag:
+ task = DummyOperator(task_id='op1')
+
+ task.test_field = template_file
+ task.template_fields = ('test_field',)
+ task.template_ext = ('.template',)
+ task.resolve_template_files()
+
+ self.assertEqual(task.test_field, '{{ ds }}')
+
+ def test_resolve_template_files_list(self):
+
+ with NamedTemporaryFile(suffix='.template') as f:
+ f = NamedTemporaryFile(suffix='.template')
+ f.write('{{ ds }}'.encode('utf8'))
+ f.flush()
+ template_dir = os.path.dirname(f.name)
+ template_file = os.path.basename(f.name)
+
+ dag = DAG('test-dag',
+ start_date=DEFAULT_DATE,
+ template_searchpath=template_dir)
+
+ with dag:
+ task = DummyOperator(task_id='op1')
+
+ task.test_field = [template_file, 'some_string']
+ task.template_fields = ('test_field',)
+ task.template_ext = ('.template',)
+ task.resolve_template_files()
+
+ self.assertEqual(task.test_field, ['{{ ds }}', 'some_string'])
+
def test_cycle(self):
# test empty
dag = DAG(
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> resolve_template_files not processing lists
> -------------------------------------------
>
> Key: AIRFLOW-3245
> URL: https://issues.apache.org/jira/browse/AIRFLOW-3245
> Project: Apache Airflow
> Issue Type: Bug
> Affects Versions: 1.10.0
> Reporter: Marcin Szymanski
> Assignee: Marcin Szymanski
> Priority: Major
> Fix For: 2.0.0
>
>
> Currently, resolve_template_files does not process template field value if it
> is a list - only strings are processed. Lists are correctly processed at a
> further stage of template parsing, and are a common use case for submitting
> multiple SQL statements
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
