This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ba645ec1154051daf8654e94ae170b49bfb6f49f Author: bolkedebruin <[email protected]> AuthorDate: Tue May 7 22:42:28 2019 +0200 [AIRFLOW-4472] Use json.dumps/loads for templating lineage data (#5253) jinja2 cannot use dict/lists as templates hence converting it to json solves this while keeping complexity down. (cherry picked from commit a6daeb544e815fe350a96d24ae3bb14aee4079a7) --- airflow/lineage/datasets.py | 11 +++++++++-- airflow/models/baseoperator.py | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/airflow/lineage/datasets.py b/airflow/lineage/datasets.py index 2602770..3d61e5d 100644 --- a/airflow/lineage/datasets.py +++ b/airflow/lineage/datasets.py @@ -16,6 +16,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import json import six from typing import List @@ -62,7 +63,11 @@ class DataSet(object): if attr in self.attributes: if self.context: env = Environment() - return env.from_string(self._data.get(attr)).render(**self.context) + # dump to json here in order to be able to manage dicts and lists + rendered = env.from_string( + json.dumps(self._data.get(attr)) + ).render(**self.context) + return json.loads(rendered) return self._data.get(attr) @@ -82,7 +87,9 @@ class DataSet(object): env = Environment() if self.context: for key, value in six.iteritems(attributes): - attributes[key] = env.from_string(value).render(**self.context) + attributes[key] = json.loads( + env.from_string(json.dumps(value)).render(**self.context) + ) d = { "typeName": self.type_name, diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index b61c980..52037c5 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -428,8 +428,8 @@ class BaseOperator(LoggingMixin): self._log = logging.getLogger("airflow.task.operators") # lineage - self.inlets = [] # type: Iterable[DataSet] - self.outlets = [] # type: Iterable[DataSet] + self.inlets = [] # type: List[DataSet] + self.outlets = [] # type: List[DataSet] self.lineage_data = None self._inlets = {
