This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f09bd4ed1bd feat: Add CLL to OpenLineage in BigQueryInsertJobOperator
(#44872)
f09bd4ed1bd is described below
commit f09bd4ed1bdb09cac3070a7c26ea90b8209e026d
Author: Kacper Muda <[email protected]>
AuthorDate: Mon Dec 30 15:43:25 2024 +0100
feat: Add CLL to OpenLineage in BigQueryInsertJobOperator (#44872)
Signed-off-by: Kacper Muda <[email protected]>
---
.../providers/google/cloud/openlineage/mixins.py | 502 +++++++++++++++------
.../providers/google/cloud/openlineage/utils.py | 115 ++++-
.../providers/google/cloud/operators/bigquery.py | 4 +-
.../tests/google/cloud/openlineage/test_mixins.py | 442 +++++++++++++++++-
.../tests/google/cloud/openlineage/test_utils.py | 131 ++++++
.../tests/google/cloud/utils/job_details.json | 17 +-
6 files changed, 1048 insertions(+), 163 deletions(-)
diff --git a/providers/src/airflow/providers/google/cloud/openlineage/mixins.py
b/providers/src/airflow/providers/google/cloud/openlineage/mixins.py
index ce7a14e03ae..8ed91424046 100644
--- a/providers/src/airflow/providers/google/cloud/openlineage/mixins.py
+++ b/providers/src/airflow/providers/google/cloud/openlineage/mixins.py
@@ -22,59 +22,65 @@ import json
import traceback
from typing import TYPE_CHECKING, cast
-if TYPE_CHECKING:
- from airflow.providers.common.compat.openlineage.facet import (
- Dataset,
- InputDataset,
- OutputDataset,
- OutputStatisticsOutputDatasetFacet,
- RunFacet,
- SchemaDatasetFacet,
- )
- from airflow.providers.google.cloud.openlineage.utils import
BigQueryJobRunFacet
+from airflow.providers.common.compat.openlineage.facet import (
+ ColumnLineageDatasetFacet,
+ ErrorMessageRunFacet,
+ ExternalQueryRunFacet,
+ Fields,
+ InputDataset,
+ InputField,
+ OutputDataset,
+ OutputStatisticsOutputDatasetFacet,
+ SchemaDatasetFacet,
+ SchemaDatasetFacetFields,
+ SQLJobFacet,
+)
+from airflow.providers.google.cloud.openlineage.utils import (
+ BIGQUERY_NAMESPACE,
+ BigQueryJobRunFacet,
+ get_from_nullable_chain,
+ merge_column_lineage_facets,
+)
+if TYPE_CHECKING:
+ from airflow.providers.common.compat.openlineage.facet import Dataset,
RunFacet
-BIGQUERY_NAMESPACE = "bigquery"
+class _BigQueryInsertJobOperatorOpenLineageMixin:
+ """Mixin for BigQueryInsertJobOperator to extract OpenLineage metadata."""
-class _BigQueryOpenLineageMixin:
def get_openlineage_facets_on_complete(self, _):
"""
- Retrieve OpenLineage data for a COMPLETE BigQuery job.
-
- This method retrieves statistics for the specified job_ids using the
BigQueryDatasetsProvider.
- It calls BigQuery API, retrieving input and output dataset info from
it, as well as run-level
- usage statistics.
+ Retrieve OpenLineage data for a completed BigQuery job.
- Run facets should contain:
- - ExternalQueryRunFacet
- - BigQueryJobRunFacet
+ This method calls BigQuery API, retrieving input and output dataset
info from it,
+ as well as run-level statistics.
Run facets may contain:
- - ErrorMessageRunFacet
+ - ExternalQueryRunFacet (for QUERY job type)
+ - BigQueryJobRunFacet
+ - ErrorMessageRunFacet (if an error occurred)
Job facets should contain:
- - SqlJobFacet if operator has self.sql
+ - SqlJobFacet (for QUERY job type)
- Input datasets should contain facets:
- - DataSourceDatasetFacet
+ Input datasets should contain:
- SchemaDatasetFacet
- Output datasets should contain facets:
- - DataSourceDatasetFacet
+ Output datasets should contain:
- SchemaDatasetFacet
- - OutputStatisticsOutputDatasetFacet
+ - OutputStatisticsOutputDatasetFacet (for QUERY job type)
+ - ColumnLineageDatasetFacet (for QUERY job type)
"""
- from airflow.providers.common.compat.openlineage.facet import
ExternalQueryRunFacet, SQLJobFacet
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.sqlparser import SQLParser
if not self.job_id:
- if hasattr(self, "log"):
- self.log.warning("No BigQuery job_id was found by
OpenLineage.")
+ self.log.warning("No BigQuery job_id was found by OpenLineage.")
# type: ignore[attr-defined]
return OperatorLineage()
if not self.hook:
+ # This can occur when in deferrable mode
from airflow.providers.google.cloud.hooks.bigquery import
BigQueryHook
self.hook = BigQueryHook(
@@ -82,64 +88,35 @@ class _BigQueryOpenLineageMixin:
impersonation_chain=self.impersonation_chain,
)
+ self.log.debug("Extracting data from bigquery job: `%s`", self.job_id)
# type: ignore[attr-defined]
+ inputs, outputs = [], []
run_facets: dict[str, RunFacet] = {
"externalQuery":
ExternalQueryRunFacet(externalQueryId=self.job_id, source="bigquery")
}
-
- job_facets = {"sql":
SQLJobFacet(query=SQLParser.normalize_sql(self.sql))}
-
- self.client = self.hook.get_client(project_id=self.hook.project_id,
location=self.location)
- job_ids = self.job_id
- if isinstance(self.job_id, str):
- job_ids = [self.job_id]
- inputs, outputs = [], []
- for job_id in job_ids:
- inner_inputs, inner_outputs, inner_run_facets =
self.get_facets(job_id=job_id)
- inputs.extend(inner_inputs)
- outputs.extend(inner_outputs)
- run_facets.update(inner_run_facets)
-
- return OperatorLineage(
- inputs=inputs,
- outputs=outputs,
- run_facets=run_facets,
- job_facets=job_facets,
- )
-
- def get_facets(self, job_id: str):
- from airflow.providers.common.compat.openlineage.facet import
ErrorMessageRunFacet
- from airflow.providers.google.cloud.openlineage.utils import
get_from_nullable_chain
-
- inputs = []
- outputs = []
- run_facets: dict[str, RunFacet] = {}
- if hasattr(self, "log"):
- self.log.debug("Extracting data from bigquery job: `%s`", job_id)
+ self._client = self.hook.get_client(project_id=self.hook.project_id,
location=self.location)
try:
- job = self.client.get_job(job_id=job_id) # type: ignore
- props = job._properties
+ job_properties =
self._client.get_job(job_id=self.job_id)._properties # type: ignore
- if get_from_nullable_chain(props, ["status", "state"]) != "DONE":
- raise ValueError(f"Trying to extract data from running
bigquery job: `{job_id}`")
+ if get_from_nullable_chain(job_properties, ["status", "state"]) !=
"DONE":
+ raise ValueError(f"Trying to extract data from running
bigquery job: `{self.job_id}`")
- run_facets["bigQueryJob"] = self._get_bigquery_job_run_facet(props)
+ run_facets.update(self._get_run_facets(job_properties))
- if get_from_nullable_chain(props, ["statistics", "numChildJobs"]):
- if hasattr(self, "log"):
- self.log.debug("Found SCRIPT job. Extracting lineage from
child jobs instead.")
+ if get_from_nullable_chain(job_properties, ["statistics",
"numChildJobs"]):
+ self.log.debug("Found SCRIPT job. Extracting lineage from
child jobs instead.") # type: ignore[attr-defined]
# SCRIPT job type has no input / output information but spawns
child jobs that have one
#
https://cloud.google.com/bigquery/docs/information-schema-jobs#multi-statement_query_job
- for child_job_id in self.client.list_jobs(parent_job=job_id):
- child_job = self.client.get_job(job_id=child_job_id) #
type: ignore
- child_inputs, child_output =
self._get_inputs_outputs_from_job(child_job._properties)
+ for child_job_id in
self._client.list_jobs(parent_job=self.job_id):
+ child_job_properties =
self._client.get_job(job_id=child_job_id)._properties # type: ignore
+ child_inputs, child_output =
self._get_inputs_and_output(child_job_properties)
inputs.extend(child_inputs)
outputs.append(child_output)
else:
- inputs, _output = self._get_inputs_outputs_from_job(props)
+ inputs, _output = self._get_inputs_and_output(job_properties)
outputs.append(_output)
+
except Exception as e:
- if hasattr(self, "log"):
- self.log.warning("Cannot retrieve job details from
BigQuery.Client. %s", e, exc_info=True)
+ self.log.warning("Cannot retrieve job details from
BigQuery.Client. %s", e, exc_info=True) # type: ignore[attr-defined]
exception_msg = traceback.format_exc()
run_facets.update(
{
@@ -149,16 +126,40 @@ class _BigQueryOpenLineageMixin:
)
}
)
- deduplicated_outputs = self._deduplicate_outputs(outputs)
- return inputs, deduplicated_outputs, run_facets
+
+ return OperatorLineage(
+ inputs=inputs,
+ outputs=self._deduplicate_outputs(outputs),
+ run_facets=run_facets,
+ job_facets={"sql":
SQLJobFacet(query=SQLParser.normalize_sql(self.sql))} if self.sql else {},
+ )
+
+ def _get_run_facets(self, properties: dict) -> dict[str, RunFacet]:
+ job_type = get_from_nullable_chain(properties, ["configuration",
"jobType"])
+
+ run_facets: dict[str, RunFacet] = {}
+ if job_type == "QUERY":
+ run_facets["bigQueryJob"] =
self._get_bigquery_job_run_facet(properties)
+
+ return run_facets
+
+ def _get_inputs_and_output(self, properties: dict) ->
tuple[list[InputDataset], OutputDataset | None]:
+ job_type = get_from_nullable_chain(properties, ["configuration",
"jobType"])
+
+ if job_type == "QUERY":
+ inputs, output =
self._get_inputs_and_output_for_query_job(properties)
+ else:
+ self.log.debug("Unsupported job type for input/output extraction:
`%s`.", job_type) # type: ignore[attr-defined]
+ inputs, output = [], None
+
+ return inputs, output
def _deduplicate_outputs(self, outputs: list[OutputDataset | None]) ->
list[OutputDataset]:
- # Sources are the same so we can compare only names
final_outputs = {}
for single_output in outputs:
if not single_output:
continue
- key = single_output.name
+ key = f"{single_output.namespace}.{single_output.name}"
if key not in final_outputs:
final_outputs[key] = single_output
continue
@@ -167,77 +168,32 @@ class _BigQueryOpenLineageMixin:
# if the rowCount or size can be summed together.
if single_output.outputFacets:
single_output.outputFacets.pop("outputStatistics", None)
- final_outputs[key] = single_output
-
- return list(final_outputs.values())
-
- def _get_inputs_outputs_from_job(
- self, properties: dict
- ) -> tuple[list[InputDataset], OutputDataset | None]:
- from airflow.providers.google.cloud.openlineage.utils import
get_from_nullable_chain
-
- input_tables = get_from_nullable_chain(properties, ["statistics",
"query", "referencedTables"]) or []
- output_table = get_from_nullable_chain(properties, ["configuration",
"query", "destinationTable"])
- inputs = [(self._get_input_dataset(input_table)) for input_table in
input_tables]
- if output_table:
- output = self._get_output_dataset(output_table)
- dataset_stat_facet = self._get_statistics_dataset_facet(properties)
- output.outputFacets = output.outputFacets or {}
- if dataset_stat_facet:
- output.outputFacets["outputStatistics"] = dataset_stat_facet
- return inputs, output
-
- @staticmethod
- def _get_bigquery_job_run_facet(properties: dict) -> BigQueryJobRunFacet:
- from airflow.providers.google.cloud.openlineage.utils import (
- BigQueryJobRunFacet,
- get_from_nullable_chain,
- )
-
- if get_from_nullable_chain(properties, ["configuration", "query",
"query"]):
- # Exclude the query to avoid event size issues and duplicating
SqlJobFacet information.
- properties = copy.deepcopy(properties)
- properties["configuration"]["query"].pop("query")
- cache_hit = get_from_nullable_chain(properties, ["statistics",
"query", "cacheHit"])
- billed_bytes = get_from_nullable_chain(properties, ["statistics",
"query", "totalBytesBilled"])
- return BigQueryJobRunFacet(
- cached=str(cache_hit).lower() == "true",
- billedBytes=int(billed_bytes) if billed_bytes else None,
- properties=json.dumps(properties),
- )
-
- @staticmethod
- def _get_statistics_dataset_facet(
- properties,
- ) -> OutputStatisticsOutputDatasetFacet | None:
- from airflow.providers.common.compat.openlineage.facet import
OutputStatisticsOutputDatasetFacet
- from airflow.providers.google.cloud.openlineage.utils import
get_from_nullable_chain
+ # If multiple outputs contain Column Level Lineage Facet - merge
the facets
+ if (
+ single_output.facets
+ and final_outputs[key].facets
+ and "columnLineage" in single_output.facets # type: ignore
+ and "columnLineage" in final_outputs[key].facets # type:
ignore
+ ):
+ single_output.facets["columnLineage"] =
merge_column_lineage_facets(
+ [
+ single_output.facets["columnLineage"], # type: ignore
+ final_outputs[key].facets["columnLineage"], # type:
ignore
+ ]
+ )
- query_plan = get_from_nullable_chain(properties, chain=["statistics",
"query", "queryPlan"])
- if not query_plan:
- return None
+ final_outputs[key] = single_output
- out_stage = query_plan[-1]
- out_rows = out_stage.get("recordsWritten", None)
- out_bytes = out_stage.get("shuffleOutputBytes", None)
- if out_bytes and out_rows:
- return OutputStatisticsOutputDatasetFacet(rowCount=int(out_rows),
size=int(out_bytes))
- return None
+ return list(final_outputs.values())
def _get_input_dataset(self, table: dict) -> InputDataset:
- from airflow.providers.common.compat.openlineage.facet import
InputDataset
-
return cast(InputDataset, self._get_dataset(table, "input"))
def _get_output_dataset(self, table: dict) -> OutputDataset:
- from airflow.providers.common.compat.openlineage.facet import
OutputDataset
-
return cast(OutputDataset, self._get_dataset(table, "output"))
def _get_dataset(self, table: dict, dataset_type: str) -> Dataset:
- from airflow.providers.common.compat.openlineage.facet import
InputDataset, OutputDataset
-
project = table.get("projectId")
dataset = table.get("datasetId")
table_name = table.get("tableId")
@@ -273,18 +229,11 @@ class _BigQueryOpenLineageMixin:
try:
return self._get_table_schema(table_name)
except Exception as e:
- if hasattr(self, "log"):
- self.log.warning("Could not extract output schema from
bigquery. %s", e)
+ self.log.warning("Could not extract output schema from bigquery.
%s", e) # type: ignore[attr-defined]
return None
def _get_table_schema(self, table: str) -> SchemaDatasetFacet | None:
- from airflow.providers.common.compat.openlineage.facet import (
- SchemaDatasetFacet,
- SchemaDatasetFacetFields,
- )
- from airflow.providers.google.cloud.openlineage.utils import
get_from_nullable_chain
-
- bq_table = self.client.get_table(table)
+ bq_table = self._client.get_table(table)
if not bq_table._properties:
return None
@@ -303,3 +252,262 @@ class _BigQueryOpenLineageMixin:
for field in fields
]
)
+
+ def _get_inputs_and_output_for_query_job(
+ self, properties: dict
+ ) -> tuple[list[InputDataset], OutputDataset | None]:
+ input_tables = get_from_nullable_chain(properties, ["statistics",
"query", "referencedTables"]) or []
+ output_table = get_from_nullable_chain(properties, ["configuration",
"query", "destinationTable"])
+
+ inputs = [
+ (self._get_input_dataset(input_table))
+ for input_table in input_tables
+ if input_table != output_table # Output table is in
`referencedTables` and needs to be removed
+ ]
+
+ if not output_table:
+ return inputs, None
+
+ output = self._get_output_dataset(output_table)
+ if dataset_stat_facet :=
self._get_statistics_dataset_facet(properties):
+ output.outputFacets = output.outputFacets or {}
+ output.outputFacets["outputStatistics"] = dataset_stat_facet
+ if cll_facet := self._get_column_level_lineage_facet(properties,
output, inputs):
+ output.facets = output.facets or {}
+ output.facets["columnLineage"] = cll_facet
+ return inputs, output
+
+ @staticmethod
+ def _get_bigquery_job_run_facet(properties: dict) -> BigQueryJobRunFacet:
+ if get_from_nullable_chain(properties, ["configuration", "query",
"query"]):
+ # Exclude the query to avoid event size issues and duplicating
SqlJobFacet information.
+ properties = copy.deepcopy(properties)
+ properties["configuration"]["query"].pop("query")
+ cache_hit = get_from_nullable_chain(properties, ["statistics",
"query", "cacheHit"])
+ billed_bytes = get_from_nullable_chain(properties, ["statistics",
"query", "totalBytesBilled"])
+ return BigQueryJobRunFacet(
+ cached=str(cache_hit).lower() == "true",
+ billedBytes=int(billed_bytes) if billed_bytes else None,
+ properties=json.dumps(properties),
+ )
+
+ @staticmethod
+ def _get_statistics_dataset_facet(
+ properties,
+ ) -> OutputStatisticsOutputDatasetFacet | None:
+ query_plan = get_from_nullable_chain(properties, chain=["statistics",
"query", "queryPlan"])
+ if not query_plan:
+ return None
+
+ out_stage = query_plan[-1]
+ out_rows = out_stage.get("recordsWritten", None)
+ out_bytes = out_stage.get("shuffleOutputBytes", None)
+ if out_bytes and out_rows:
+ return OutputStatisticsOutputDatasetFacet(rowCount=int(out_rows),
size=int(out_bytes))
+ return None
+
+ def _get_column_level_lineage_facet(
+ self, properties: dict, output: OutputDataset, inputs:
list[InputDataset]
+ ) -> ColumnLineageDatasetFacet | None:
+ """
+ Extract column-level lineage information from a BigQuery job and
return it as a facet.
+
+ The Column Level Lineage Facet will NOT be returned if any of the
following condition is met:
+ - The parsed result does not contain column lineage information.
+ - The parsed result does not contain exactly one output table.
+ - The parsed result has a different output table than the output table
from the BQ job.
+ - The parsed result has at least one input table not present in the
input tables from the BQ job.
+ - The parsed result has a column not present in the schema of given
dataset from the BQ job.
+
+ Args:
+ properties: The properties of the BigQuery job.
+ output: The output dataset for which the column lineage is being
extracted.
+
+ Returns:
+ The extracted Column Lineage Dataset Facet, or None if conditions
are not met.
+ """
+ from airflow.providers.openlineage.sqlparser import SQLParser
+
+ # Extract SQL query and parse it
+ self.log.debug("Extracting column-level lineage facet from BigQuery
query.") # type: ignore[attr-defined]
+ query = get_from_nullable_chain(properties, ["configuration", "query",
"query"]) or ""
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string(SQLParser.normalize_sql(query)))
+
+ if parse_result is None or parse_result.column_lineage == []:
+ self.log.debug("No column-level lineage found in the SQL query.
Facet generation skipped.") # type: ignore[attr-defined]
+ return None
+
+ default_dataset, default_project =
self._extract_default_dataset_and_project(
+ properties,
+ self.project_id, # type: ignore[attr-defined]
+ )
+
+ # Verify if the output table id from the parse result matches the BQ
job output table
+ if not self._validate_output_table_id(
+ parse_result,
+ output,
+ default_project,
+ default_dataset,
+ ):
+ return None
+
+ # Verify if all columns from parse results are present in the output
dataset schema
+ if not self._validate_output_columns(parse_result, output):
+ return None
+
+ input_tables_from_parse_result = self._extract_parsed_input_tables(
+ parse_result, default_project, default_dataset
+ )
+ input_tables_from_bq = {input_ds.name:
self._extract_column_names(input_ds) for input_ds in inputs}
+
+ # Verify if all datasets from parse results are present in bq job
input datasets
+ if not self._validate_input_tables(input_tables_from_parse_result,
input_tables_from_bq):
+ return None
+
+ # Verify if all columns from parse results are present in their
respective bq job input datasets
+ if not self._validate_input_columns(input_tables_from_parse_result,
input_tables_from_bq):
+ return None
+
+ return self._generate_column_lineage_facet(parse_result,
default_project, default_dataset)
+
+ @staticmethod
+ def _get_qualified_name_from_parse_result(table, default_project: str,
default_dataset: str) -> str:
+ """Get the qualified name of a table from the parse result."""
+ return ".".join(
+ (
+ table.database or default_project,
+ table.schema or default_dataset,
+ table.name,
+ )
+ )
+
+ @staticmethod
+ def _extract_default_dataset_and_project(properties: dict,
default_project: str) -> tuple[str, str]:
+ """Extract the default dataset and project from the BigQuery job
properties."""
+ default_dataset_obj = get_from_nullable_chain(
+ properties, ["configuration", "query", "defaultDataset"]
+ )
+ default_dataset = default_dataset_obj.get("datasetId", "") if
default_dataset_obj else ""
+ default_project = (
+ default_dataset_obj.get("projectId", default_project) if
default_dataset_obj else default_project
+ )
+ return default_dataset, default_project
+
+ def _validate_output_table_id(
+ self, parse_result, output: OutputDataset, default_project: str,
default_dataset: str
+ ) -> bool:
+ """Check if the output table id from the parse result matches the BQ
job output table."""
+ if len(parse_result.out_tables) != 1:
+ self.log.debug( # type: ignore[attr-defined]
+ "Invalid output tables in the parse result: `%s`. Expected
exactly one output table.",
+ parse_result.out_tables,
+ )
+ return False
+
+ parsed_output_table = self._get_qualified_name_from_parse_result(
+ parse_result.out_tables[0], default_project, default_dataset
+ )
+ if parsed_output_table != output.name:
+ self.log.debug( # type: ignore[attr-defined]
+ "Mismatch between parsed output table `%s` and BQ job output
table `%s`.",
+ parsed_output_table,
+ output.name,
+ )
+ return False
+ return True
+
+ @staticmethod
+ def _extract_column_names(dataset: Dataset) -> list[str]:
+ """Extract column names from a dataset's schema."""
+ return [
+ f.name
+ for f in dataset.facets.get("schema",
SchemaDatasetFacet(fields=[])).fields # type: ignore[union-attr]
+ if dataset.facets
+ ]
+
+ def _validate_output_columns(self, parse_result, output: OutputDataset) ->
bool:
+ """Validate if all descendant columns in parse result exist in output
dataset schema."""
+ output_column_names = self._extract_column_names(output)
+ missing_columns = [
+ lineage.descendant.name
+ for lineage in parse_result.column_lineage
+ if lineage.descendant.name not in output_column_names
+ ]
+ if missing_columns:
+ self.log.debug( # type: ignore[attr-defined]
+ "Output dataset schema is missing columns from the parse
result: `%s`.", missing_columns
+ )
+ return False
+ return True
+
+ def _extract_parsed_input_tables(
+ self, parse_result, default_project: str, default_dataset: str
+ ) -> dict[str, list[str]]:
+ """Extract input tables and their columns from the parse result."""
+ input_tables: dict[str, list[str]] = {}
+ for lineage in parse_result.column_lineage:
+ for column_meta in lineage.lineage:
+ if not column_meta.origin:
+ self.log.debug( # type: ignore[attr-defined]
+ "Column `%s` lacks origin information. Skipping facet
generation.", column_meta.name
+ )
+ return {}
+
+ input_table_id = self._get_qualified_name_from_parse_result(
+ column_meta.origin, default_project, default_dataset
+ )
+ input_tables.setdefault(input_table_id,
[]).append(column_meta.name)
+ return input_tables
+
+ def _validate_input_tables(
+ self, parsed_input_tables: dict[str, list[str]], input_tables_from_bq:
dict[str, list[str]]
+ ) -> bool:
+ """Validate if all parsed input tables exist in the BQ job's input
datasets."""
+ if not parsed_input_tables:
+ self.log.debug("No input tables found in the parse result. Facet
generation skipped.") # type: ignore[attr-defined]
+ return False
+ if missing_tables := set(parsed_input_tables) -
set(input_tables_from_bq):
+ self.log.debug( # type: ignore[attr-defined]
+ "Parsed input tables not found in the BQ job's input datasets:
`%s`.", missing_tables
+ )
+ return False
+ return True
+
+ def _validate_input_columns(
+ self, parsed_input_tables: dict[str, list[str]], input_tables_from_bq:
dict[str, list[str]]
+ ) -> bool:
+ """Validate if all parsed input columns exist in their respective BQ
job input table schemas."""
+ if not parsed_input_tables:
+ self.log.debug("No input tables found in the parse result. Facet
generation skipped.") # type: ignore[attr-defined]
+ return False
+ for table, columns in parsed_input_tables.items():
+ if missing_columns := set(columns) -
set(input_tables_from_bq.get(table, [])):
+ self.log.debug( # type: ignore[attr-defined]
+ "Input table `%s` is missing columns from the parse
result: `%s`.", table, missing_columns
+ )
+ return False
+ return True
+
+ def _generate_column_lineage_facet(
+ self, parse_result, default_project: str, default_dataset: str
+ ) -> ColumnLineageDatasetFacet:
+ """Generate the ColumnLineageDatasetFacet based on the parsed
result."""
+ return ColumnLineageDatasetFacet(
+ fields={
+ lineage.descendant.name: Fields(
+ inputFields=[
+ InputField(
+ namespace=BIGQUERY_NAMESPACE,
+ name=self._get_qualified_name_from_parse_result(
+ column_meta.origin, default_project,
default_dataset
+ ),
+ field=column_meta.name,
+ )
+ for column_meta in lineage.lineage
+ ],
+ transformationType="",
+ transformationDescription="",
+ )
+ for lineage in parse_result.column_lineage
+ }
+ )
diff --git a/providers/src/airflow/providers/google/cloud/openlineage/utils.py
b/providers/src/airflow/providers/google/cloud/openlineage/utils.py
index 1700ff29619..0f3dcb5d4be 100644
--- a/providers/src/airflow/providers/google/cloud/openlineage/utils.py
+++ b/providers/src/airflow/providers/google/cloud/openlineage/utils.py
@@ -21,16 +21,10 @@ import copy
import logging
import os
import pathlib
+from collections import defaultdict
from typing import TYPE_CHECKING, Any
from attr import define, field
-
-if TYPE_CHECKING:
- from google.cloud.bigquery.table import Table
-
- from airflow.providers.common.compat.openlineage.facet import Dataset
- from airflow.utils.context import Context
-
from google.cloud.dataproc_v1 import Batch, RuntimeConfig
from airflow.providers.common.compat.openlineage.facet import (
@@ -51,6 +45,13 @@ from airflow.providers.common.compat.openlineage.utils.spark
import (
from airflow.providers.google import __version__ as provider_version
from airflow.providers.google.cloud.hooks.gcs import _parse_gcs_url
+if TYPE_CHECKING:
+ from google.cloud.bigquery.table import Table
+
+ from airflow.providers.common.compat.openlineage.facet import Dataset
+ from airflow.utils.context import Context
+
+
log = logging.getLogger(__name__)
BIGQUERY_NAMESPACE = "bigquery"
@@ -58,6 +59,106 @@ BIGQUERY_URI = "bigquery"
WILDCARD = "*"
+def merge_column_lineage_facets(facets: list[ColumnLineageDatasetFacet]) ->
ColumnLineageDatasetFacet:
+ """
+ Merge multiple column lineage facets into a single consolidated facet.
+
+ Specifically, it aggregates input fields and transformations for each
field across all provided facets.
+
+ Args:
+ facets: Column Lineage Facets to be merged.
+
+ Returns:
+ A new Column Lineage Facet containing all fields, their respective
input fields and transformations.
+
+ Notes:
+ - Input fields are uniquely identified by their `(namespace, name,
field)` tuple.
+ - If multiple facets contain the same field with the same input field,
those input
+ fields are merged without duplication.
+ - Transformations associated with input fields are also merged. If
transformations
+ are not supported by the version of the `InputField` class, they
will be omitted.
+ - Transformation merging relies on a composite key of the field name
and input field
+ tuple to track and consolidate transformations.
+
+ Examples:
+ Case 1: Two facets with the same input field
+ ```
+ >>> facet1 = ColumnLineageDatasetFacet(
+ ... fields={"columnA":
Fields(inputFields=[InputField("namespace1", "dataset1", "field1")])}
+ ... )
+ >>> facet2 = ColumnLineageDatasetFacet(
+ ... fields={"columnA":
Fields(inputFields=[InputField("namespace1", "dataset1", "field1")])}
+ ... )
+ >>> merged = merge_column_lineage_facets([facet1, facet2])
+ >>> merged.fields["columnA"].inputFields
+ [InputField("namespace1", "dataset1", "field1")]
+ ```
+
+ Case 2: Two facets with different transformations for the same input
field
+ ```
+ >>> facet1 = ColumnLineageDatasetFacet(
+ ... fields={
+ ... "columnA": Fields(
+ ... inputFields=[InputField("namespace1", "dataset1",
"field1", transformations=["t1"])]
+ ... )
+ ... }
+ ... )
+ >>> facet2 = ColumnLineageDatasetFacet(
+ ... fields={
+ ... "columnA": Fields(
+ ... inputFields=[InputField("namespace1", "dataset1",
"field1", transformations=["t2"])]
+ ... )
+ ... }
+ ... )
+ >>> merged = merge_column_lineage_facets([facet1, facet2])
+ >>> merged.fields["columnA"].inputFields[0].transformations
+ ["t1", "t2"]
+ ```
+ """
+ # Dictionary to collect all unique input fields for each field name
+ fields_sources: dict[str, set[tuple[str, str, str]]] = defaultdict(set)
+ # Dictionary to aggregate transformations for each input field
+ transformations: dict[str, list] = defaultdict(list)
+
+ for facet in facets:
+ for field_name, single_field in facet.fields.items():
+ for input_field in single_field.inputFields:
+ input_key_fields = (input_field.namespace, input_field.name,
input_field.field)
+ fields_sources[field_name].add(input_key_fields)
+
+ if single_transformations := getattr(input_field,
"transformations", []):
+ transformation_key = "".join((field_name,
*input_key_fields))
+
transformations[transformation_key].extend(single_transformations)
+
+ # Check if the `InputField` class supports the `transformations` attribute
(since OL client 1.17.1)
+ input_field_allows_transformation_info = True
+ try:
+ InputField(namespace="a", name="b", field="c", transformations=[])
+ except TypeError:
+ input_field_allows_transformation_info = False
+
+ return ColumnLineageDatasetFacet(
+ fields={
+ field_name: Fields(
+ inputFields=[
+ InputField(
+ namespace,
+ name,
+ column,
+ transformations.get("".join((field_name, namespace,
name, column)), []),
+ )
+ if input_field_allows_transformation_info
+ else InputField(namespace, name, column)
+ for namespace, name, column in sorted(input_fields)
+ ],
+ transformationType="", # Legacy transformation information
+ transformationDescription="", # Legacy transformation
information
+ )
+ for field_name, input_fields in fields_sources.items()
+ }
+ )
+
+
def extract_ds_name_from_gcs_path(path: str) -> str:
"""
Extract and process the dataset name from a given path.
diff --git a/providers/src/airflow/providers/google/cloud/operators/bigquery.py
b/providers/src/airflow/providers/google/cloud/operators/bigquery.py
index 43cf1a8f771..bc950ec9f3f 100644
--- a/providers/src/airflow/providers/google/cloud/operators/bigquery.py
+++ b/providers/src/airflow/providers/google/cloud/operators/bigquery.py
@@ -48,7 +48,7 @@ from airflow.providers.google.cloud.links.bigquery import (
BigQueryJobDetailLink,
BigQueryTableLink,
)
-from airflow.providers.google.cloud.openlineage.mixins import
_BigQueryOpenLineageMixin
+from airflow.providers.google.cloud.openlineage.mixins import
_BigQueryInsertJobOperatorOpenLineageMixin
from airflow.providers.google.cloud.operators.cloud_base import
GoogleCloudBaseOperator
from airflow.providers.google.cloud.triggers.bigquery import (
BigQueryCheckTrigger,
@@ -2491,7 +2491,7 @@ class
BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
return table
-class BigQueryInsertJobOperator(GoogleCloudBaseOperator,
_BigQueryOpenLineageMixin):
+class BigQueryInsertJobOperator(GoogleCloudBaseOperator,
_BigQueryInsertJobOperatorOpenLineageMixin):
"""
Execute a BigQuery job.
diff --git a/providers/tests/google/cloud/openlineage/test_mixins.py
b/providers/tests/google/cloud/openlineage/test_mixins.py
index fb047ddc2d1..824da4b8447 100644
--- a/providers/tests/google/cloud/openlineage/test_mixins.py
+++ b/providers/tests/google/cloud/openlineage/test_mixins.py
@@ -16,24 +16,93 @@
# under the License.
from __future__ import annotations
+import copy
import json
+import logging
import os
from unittest.mock import MagicMock
import pytest
from airflow.providers.common.compat.openlineage.facet import (
+ ColumnLineageDatasetFacet,
+ Dataset,
ExternalQueryRunFacet,
+ Fields,
InputDataset,
+ InputField,
OutputDataset,
OutputStatisticsOutputDatasetFacet,
SchemaDatasetFacet,
SchemaDatasetFacetFields,
)
-from airflow.providers.google.cloud.openlineage.mixins import
_BigQueryOpenLineageMixin
+from airflow.providers.google.cloud.openlineage.mixins import
_BigQueryInsertJobOperatorOpenLineageMixin
from airflow.providers.google.cloud.openlineage.utils import (
BigQueryJobRunFacet,
)
+from airflow.providers.openlineage.sqlparser import SQLParser
+
+QUERY_JOB_PROPERTIES = {
+ "configuration": {
+ "query": {
+ "query": """
+ INSERT INTO dest_project.dest_dataset.dest_table
+ SELECT a, b, c FROM source_project.source_dataset.source_table
+ UNION ALL
+ SELECT a, b, c FROM source_table2
+ """,
+ "defaultDataset": {"datasetId": "default_dataset", "projectId":
"default_project"},
+ }
+ }
+}
+OUTPUT_DATASET = OutputDataset(
+ namespace="bigquery",
+ name="dest_project.dest_dataset.dest_table",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("a", "STRING"),
+ SchemaDatasetFacetFields("b", "STRING"),
+ SchemaDatasetFacetFields("c", "STRING"),
+ SchemaDatasetFacetFields("d", "STRING"),
+ SchemaDatasetFacetFields("e", "STRING"),
+ SchemaDatasetFacetFields("f", "STRING"),
+ SchemaDatasetFacetFields("g", "STRING"),
+ ]
+ )
+ },
+)
+INPUT_DATASETS = [
+ InputDataset(
+ namespace="bigquery",
+ name="source_project.source_dataset.source_table",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("a", "STRING"),
+ SchemaDatasetFacetFields("b", "STRING"),
+ SchemaDatasetFacetFields("c", "STRING"),
+ SchemaDatasetFacetFields("x", "STRING"),
+ ]
+ )
+ },
+ ),
+ InputDataset(
+ namespace="bigquery",
+ name="default_project.default_dataset.source_table2",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("a", "STRING"),
+ SchemaDatasetFacetFields("b", "STRING"),
+ SchemaDatasetFacetFields("c", "STRING"),
+ SchemaDatasetFacetFields("y", "STRING"),
+ ]
+ )
+ },
+ ),
+ InputDataset("bigquery", "some.random.tb"),
+]
def read_common_json_file(rel: str):
@@ -61,10 +130,12 @@ class TestBigQueryOpenLineageMixin:
hook = MagicMock()
self.client = MagicMock()
- class BQOperator(_BigQueryOpenLineageMixin):
+ class BQOperator(_BigQueryInsertJobOperatorOpenLineageMixin):
sql = ""
job_id = "job_id"
+ project_id = "project_id"
location = None
+ log = logging.getLogger("BQOperator")
@property
def hook(self):
@@ -202,6 +273,100 @@ class TestBigQueryOpenLineageMixin:
assert second_result.name == "d2"
assert second_result.facets == {"t20": "t20"}
+ def test_deduplicate_outputs_with_cll(self):
+ outputs = [
+ None,
+ OutputDataset(
+ name="a.b.c",
+ namespace="bigquery",
+ facets={
+ "columnLineage": ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.1",
"c")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "d": Fields(
+ inputFields=[InputField("bigquery", "a.b.2",
"d")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ )
+ },
+ ),
+ OutputDataset(
+ name="a.b.c",
+ namespace="bigquery",
+ facets={
+ "columnLineage": ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.3",
"x")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "e": Fields(
+ inputFields=[InputField("bigquery", "a.b.1",
"e")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ )
+ },
+ ),
+ OutputDataset(
+ name="x.y.z",
+ namespace="bigquery",
+ facets={
+ "columnLineage": ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.3",
"x")],
+ transformationType="",
+ transformationDescription="",
+ )
+ }
+ )
+ },
+ ),
+ ]
+ result = self.operator._deduplicate_outputs(outputs)
+ assert len(result) == 2
+ first_result = result[0]
+ assert first_result.name == "a.b.c"
+ assert first_result.facets["columnLineage"] ==
ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.1", "c"),
InputField("bigquery", "a.b.3", "x")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "d": Fields(
+ inputFields=[InputField("bigquery", "a.b.2", "d")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "e": Fields(
+ inputFields=[InputField("bigquery", "a.b.1", "e")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ )
+ second_result = result[1]
+ assert second_result.name == "x.y.z"
+ assert second_result.facets["columnLineage"] ==
ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.3", "x")],
+ transformationType="",
+ transformationDescription="",
+ )
+ }
+ )
+
@pytest.mark.parametrize("cache", (None, "false", False, 0))
def test_get_job_run_facet_no_cache_and_with_bytes(self, cache):
properties = {
@@ -259,3 +424,276 @@ class TestBigQueryOpenLineageMixin:
result = self.operator._get_statistics_dataset_facet(properties)
assert result.rowCount == 123
assert result.size == 321
+
+ def test_get_column_level_lineage_facet(self):
+ result = self.operator._get_column_level_lineage_facet(
+ QUERY_JOB_PROPERTIES, OUTPUT_DATASET, INPUT_DATASETS
+ )
+ assert result == ColumnLineageDatasetFacet(
+ fields={
+ col: Fields(
+ inputFields=[
+ InputField("bigquery",
"default_project.default_dataset.source_table2", col),
+ InputField("bigquery",
"source_project.source_dataset.source_table", col),
+ ],
+ transformationType="",
+ transformationDescription="",
+ )
+ for col in ("a", "b", "c")
+ }
+ )
+
+ def
test_get_column_level_lineage_facet_early_exit_empty_cll_from_parser(self):
+ properties = {"configuration": {"query": {"query": "SELECT 1"}}}
+ assert (
+ self.operator._get_column_level_lineage_facet(properties,
OUTPUT_DATASET, INPUT_DATASETS) is None
+ )
+ assert self.operator._get_column_level_lineage_facet({},
OUTPUT_DATASET, INPUT_DATASETS) is None
+
+ def
test_get_column_level_lineage_facet_early_exit_output_table_id_mismatch(self):
+ output = copy.deepcopy(OUTPUT_DATASET)
+ output.name = "different.name.table"
+ assert (
+
self.operator._get_column_level_lineage_facet(QUERY_JOB_PROPERTIES, output,
INPUT_DATASETS)
+ is None
+ )
+
+ def
test_get_column_level_lineage_facet_early_exit_output_columns_mismatch(self):
+ output = copy.deepcopy(OUTPUT_DATASET)
+ output.facets["schema"].fields = [
+ SchemaDatasetFacetFields("different_col", "STRING"),
+ ]
+ assert (
+
self.operator._get_column_level_lineage_facet(QUERY_JOB_PROPERTIES, output,
INPUT_DATASETS)
+ is None
+ )
+
+ def
test_get_column_level_lineage_facet_early_exit_wrong_parsed_input_tables(self):
+ properties = {
+ "configuration": {
+ "query": {
+ "query": """
+ INSERT INTO dest_project.dest_dataset.dest_table
+ SELECT a, b, c FROM some.wrong.source_table
+ """,
+ }
+ }
+ }
+ assert (
+ self.operator._get_column_level_lineage_facet(properties,
OUTPUT_DATASET, INPUT_DATASETS) is None
+ )
+
+ def
test_get_column_level_lineage_facet_early_exit_wrong_parsed_input_columns(self):
+ properties = {
+ "configuration": {
+ "query": {
+ "query": """
+ INSERT INTO dest_project.dest_dataset.dest_table
+ SELECT wrong_col, wrong2, wrong3 FROM
source_project.source_dataset.source_table
+ """,
+ }
+ }
+ }
+ assert (
+ self.operator._get_column_level_lineage_facet(properties,
OUTPUT_DATASET, INPUT_DATASETS) is None
+ )
+
+ def test_get_qualified_name_from_parse_result(self):
+ class _Table: # Replacement for SQL parser TableMeta
+ database = "project"
+ schema = "dataset"
+ name = "table"
+
+ class _TableNoSchema: # Replacement for SQL parser TableMeta
+ database = None
+ schema = "dataset"
+ name = "table"
+
+ class _TableNoSchemaNoDb: # Replacement for SQL parser TableMeta
+ database = None
+ schema = None
+ name = "table"
+
+ result = self.operator._get_qualified_name_from_parse_result(
+ table=_Table(),
+ default_project="default_project",
+ default_dataset="default_dataset",
+ )
+ assert result == "project.dataset.table"
+
+ result = self.operator._get_qualified_name_from_parse_result(
+ table=_TableNoSchema(),
+ default_project="default_project",
+ default_dataset="default_dataset",
+ )
+ assert result == "default_project.dataset.table"
+
+ result = self.operator._get_qualified_name_from_parse_result(
+ table=_TableNoSchemaNoDb(),
+ default_project="default_project",
+ default_dataset="default_dataset",
+ )
+ assert result == "default_project.default_dataset.table"
+
+ def test_extract_default_dataset_and_project(self):
+ properties = {"configuration": {"query": {"defaultDataset":
{"datasetId": "default_dataset"}}}}
+ result =
self.operator._extract_default_dataset_and_project(properties,
"default_project")
+ assert result == ("default_dataset", "default_project")
+
+ properties = {
+ "configuration": {
+ "query": {"defaultDataset": {"datasetId": "default_dataset",
"projectId": "default_project"}}
+ }
+ }
+ result =
self.operator._extract_default_dataset_and_project(properties,
"another_project")
+ assert result == ("default_dataset", "default_project")
+
+ result = self.operator._extract_default_dataset_and_project({},
"default_project")
+ assert result == ("", "default_project")
+
+ def test_validate_output_table_id_no_table(self):
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("SELECT 1"))
+ assert parse_result.out_tables == []
+ assert self.operator._validate_output_table_id(parse_result, None,
None, None) is False
+
+ def test_validate_output_table_id_multiple_tables(self):
+ query = "INSERT INTO a.b.c VALUES (1); INSERT INTO d.e.f VALUES (2);"
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string(query))
+ assert len(parse_result.out_tables) == 2
+ assert self.operator._validate_output_table_id(parse_result, None,
None, None) is False
+
+ def test_validate_output_table_id_mismatch(self):
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("INSERT INTO a.b.c
VALUES (1)"))
+ assert len(parse_result.out_tables) == 1
+ assert parse_result.out_tables[0].qualified_name == "a.b.c"
+ assert (
+ self.operator._validate_output_table_id(parse_result,
OutputDataset("", "d.e.f"), None, None)
+ is False
+ )
+
+ def test_validate_output_table_id(self):
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("INSERT INTO a.b.c
VALUES (1)"))
+ assert len(parse_result.out_tables) == 1
+ assert parse_result.out_tables[0].qualified_name == "a.b.c"
+ assert (
+ self.operator._validate_output_table_id(parse_result,
OutputDataset("", "a.b.c"), None, None)
+ is True
+ )
+
+ def test_validate_output_table_id_query_with_table_name_only(self):
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("INSERT INTO c VALUES
(1)"))
+ assert len(parse_result.out_tables) == 1
+ assert parse_result.out_tables[0].qualified_name == "c"
+ assert (
+ self.operator._validate_output_table_id(parse_result,
OutputDataset("", "a.b.c"), "a", "b")
+ is True
+ )
+
+ def test_extract_column_names_dataset_without_schema(self):
+ assert self.operator._extract_column_names(Dataset("a", "b")) == []
+
+ def test_extract_column_names_dataset_(self):
+ ds = Dataset(
+ "a",
+ "b",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("col1", "STRING"),
+ SchemaDatasetFacetFields("col2", "STRING"),
+ ]
+ )
+ },
+ )
+ assert self.operator._extract_column_names(ds) == ["col1", "col2"]
+
+ def test_validate_output_columns_mismatch(self):
+ ds = OutputDataset(
+ "a",
+ "b",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("col1", "STRING"),
+ SchemaDatasetFacetFields("col2", "STRING"),
+ ]
+ )
+ },
+ )
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("SELECT a , b FROM c"))
+ assert self.operator._validate_output_columns(parse_result, ds) is
False
+
+ def test_validate_output_columns(self):
+ ds = OutputDataset(
+ "a",
+ "b",
+ facets={
+ "schema": SchemaDatasetFacet(
+ fields=[
+ SchemaDatasetFacetFields("a", "STRING"),
+ SchemaDatasetFacetFields("b", "STRING"),
+ ]
+ )
+ },
+ )
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("SELECT a , b FROM c"))
+ assert self.operator._validate_output_columns(parse_result, ds) is True
+
+ def test_extract_parsed_input_tables(self):
+ query = "INSERT INTO x SELECT a, b from project1.ds1.tb1; INSERT INTO
y SELECT c, d from tb2;"
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string(query))
+ assert self.operator._extract_parsed_input_tables(parse_result,
"default_project", "default_ds") == {
+ "project1.ds1.tb1": ["a", "b"],
+ "default_project.default_ds.tb2": ["c", "d"],
+ }
+
+ def test_extract_parsed_input_tables_no_cll(self):
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string("SELECT 1"))
+ assert self.operator._extract_parsed_input_tables(parse_result, "p",
"d") == {}
+
+ def test_validate_input_tables_mismatch(self):
+ result = self.operator._validate_input_tables({"a": None, "b": None},
{"a": None, "c": None})
+ assert result is False
+
+ def test_validate_input_tables_bq_has_more_tables(self):
+ result = self.operator._validate_input_tables({"a": None}, {"a": None,
"c": None})
+ assert result is True
+
+ def test_validate_input_tables_empty(self):
+ result = self.operator._validate_input_tables({}, {"a": None, "c":
None})
+ assert result is False
+
+ def test_validate_input_columns_mismatch(self):
+ result = self.operator._validate_input_columns(
+ {"a": ["1", "2"], "b": ["3", "4"]}, {"a": ["1", "2", "3"], "c":
["4", "5"]}
+ )
+ assert result is False
+
+ def test_validate_input_columns_bq_has_more_cols(self):
+ result = self.operator._validate_input_columns(
+ {"a": ["1", "2"]}, {"a": ["1", "2", "3"], "c": ["4", "5"]}
+ )
+ assert result is True
+
+ def test_validate_input_columns_empty(self):
+ result = self.operator._validate_input_columns({}, {"a": ["1", "2",
"3"], "c": ["4", "5"]})
+ assert result is False
+
+ def test_generate_column_lineage_facet(self):
+ query = "INSERT INTO b.c SELECT c, d from tb2;"
+ parse_result =
SQLParser("bigquery").parse(SQLParser.split_sql_string(query))
+ result = self.operator._generate_column_lineage_facet(parse_result,
"default_project", "default_ds")
+ assert result == ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery",
"default_project.default_ds.tb2", "c")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "d": Fields(
+ inputFields=[InputField("bigquery",
"default_project.default_ds.tb2", "d")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ )
diff --git a/providers/tests/google/cloud/openlineage/test_utils.py
b/providers/tests/google/cloud/openlineage/test_utils.py
index 86f87531d24..58949125f84 100644
--- a/providers/tests/google/cloud/openlineage/test_utils.py
+++ b/providers/tests/google/cloud/openlineage/test_utils.py
@@ -23,6 +23,7 @@ from unittest.mock import MagicMock, patch
import pytest
from google.cloud.bigquery.table import Table
from google.cloud.dataproc_v1 import Batch, RuntimeConfig
+from openlineage.client.facet_v2 import column_lineage_dataset
from airflow.providers.common.compat.openlineage.facet import (
ColumnLineageDatasetFacet,
@@ -47,6 +48,7 @@ from airflow.providers.google.cloud.openlineage.utils import (
get_identity_column_lineage_facet,
inject_openlineage_properties_into_dataproc_batch,
inject_openlineage_properties_into_dataproc_job,
+ merge_column_lineage_facets,
)
TEST_DATASET = "test-dataset"
@@ -91,6 +93,135 @@ class TableMock(MagicMock):
return self.inputs.pop()
+def test_merge_column_lineage_facets():
+ result = merge_column_lineage_facets(
+ [
+ ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[
+ InputField(
+ "bigquery",
+ "a.b.1",
+ "c",
+ [
+ column_lineage_dataset.Transformation(
+ "type", "some_subtype", "desc", False
+ )
+ ],
+ )
+ ],
+ transformationType="IDENTITY",
+ transformationDescription="IDENTICAL",
+ ),
+ "d": Fields(
+ inputFields=[
+ InputField(
+ "bigquery",
+ "a.b.2",
+ "d",
+ [column_lineage_dataset.Transformation("t",
"s", "d", False)],
+ )
+ ],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ ),
+ ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[
+ InputField(
+ "bigquery",
+ "a.b.3",
+ "x",
+ [
+ column_lineage_dataset.Transformation(
+ "another_type", "different_subtype",
"example", True
+ )
+ ],
+ ),
+ InputField(
+ "bigquery",
+ "a.b.1",
+ "c",
+ [
+ column_lineage_dataset.Transformation(
+ "diff_type", "diff_subtype", "diff",
True
+ )
+ ],
+ ),
+ ],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "e": Fields(
+ inputFields=[InputField("bigquery", "a.b.1", "e")],
+ transformationType="IDENTITY",
+ transformationDescription="IDENTICAL",
+ ),
+ }
+ ),
+ ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[InputField("bigquery", "a.b.3", "x")],
+ transformationType="",
+ transformationDescription="",
+ )
+ }
+ ),
+ ]
+ )
+ assert result == ColumnLineageDatasetFacet(
+ fields={
+ "c": Fields(
+ inputFields=[
+ InputField(
+ "bigquery",
+ "a.b.1",
+ "c",
+ [
+ column_lineage_dataset.Transformation("type",
"some_subtype", "desc", False),
+ column_lineage_dataset.Transformation("diff_type",
"diff_subtype", "diff", True),
+ ],
+ ),
+ InputField(
+ "bigquery",
+ "a.b.3",
+ "x",
+ [
+ column_lineage_dataset.Transformation(
+ "another_type", "different_subtype",
"example", True
+ )
+ ],
+ ),
+ ],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "d": Fields(
+ inputFields=[
+ InputField(
+ "bigquery",
+ "a.b.2",
+ "d",
+ [column_lineage_dataset.Transformation("t", "s", "d",
False)],
+ )
+ ],
+ transformationType="",
+ transformationDescription="",
+ ),
+ "e": Fields(
+ inputFields=[InputField("bigquery", "a.b.1", "e")],
+ transformationType="",
+ transformationDescription="",
+ ),
+ }
+ )
+
+
def test_get_facets_from_bq_table():
expected_facets = {
"schema": SchemaDatasetFacet(
diff --git a/providers/tests/google/cloud/utils/job_details.json
b/providers/tests/google/cloud/utils/job_details.json
index f12ec1321d5..b533e0faa67 100644
--- a/providers/tests/google/cloud/utils/job_details.json
+++ b/providers/tests/google/cloud/utils/job_details.json
@@ -225,11 +225,18 @@
"billingTier": 1,
"totalSlotMs": "825",
"cacheHit": false,
- "referencedTables": [{
- "projectId": "airflow-openlineage",
- "datasetId": "new_dataset",
- "tableId": "test_table"
- }],
+ "referencedTables": [
+ {
+ "projectId": "airflow-openlineage",
+ "datasetId": "new_dataset",
+ "tableId": "test_table"
+ },
+ {
+ "projectId": "airflow-openlineage",
+ "datasetId": "new_dataset",
+ "tableId": "output_table"
+ }
+ ],
"statementType": "SELECT"
},
"totalSlotMs": "825"