nailo2c commented on code in PR #69234:
URL: https://github.com/apache/airflow/pull/69234#discussion_r3557992780
##########
providers/google/src/airflow/providers/google/cloud/openlineage/mixins.py:
##########
@@ -109,14 +110,50 @@ def get_openlineage_facets_on_complete(self, _):
run_facets["bigQueryJob"] =
self._get_bigquery_job_run_facet(job_properties)
if get_from_nullable_chain(job_properties, ["statistics",
"numChildJobs"]):
- self.log.debug("Found SCRIPT job. Extracting lineage from
child jobs instead.")
- # SCRIPT job type has no input / output information but spawns
child jobs that have one
+ self.log.debug("Found SCRIPT job. Extracting lineage from
child jobs.")
+ # SCRIPT job has no input/output of its own but spawns child
jobs that do. The parent
+ # task keeps the aggregated coarse-grained lineage (backward
compatible) while each
+ # child query is additionally emitted below as its own event
for per-statement detail.
#
https://cloud.google.com/bigquery/docs/information-schema-jobs#multi-statement_query_job
- for child_job_id in
self._client.list_jobs(parent_job=self.job_id):
- child_job_properties =
self._client.get_job(job_id=child_job_id)._properties
- child_inputs, child_outputs =
self._get_inputs_and_outputs(child_job_properties)
+ child_jobs_to_emit = []
+ for child_job in
self._client.list_jobs(parent_job=self.job_id):
+ # BigQuery returns Job objects; keep raw job IDs supported
for lightweight clients.
+ child_job_id = getattr(child_job, "job_id", child_job)
+ try:
+ child_job_properties =
self._client.get_job(job_id=child_job_id)._properties
+ child_inputs, child_outputs =
self._get_inputs_and_outputs(child_job_properties)
+ except Exception as child_exception:
+ self.log.warning(
+ "Cannot retrieve lineage for BigQuery child job
`%s`. %s",
+ child_job_id,
+ child_exception,
+ exc_info=True,
+ )
+ continue
inputs.extend(child_inputs)
outputs.extend(child_outputs)
+ child_jobs_to_emit.append(
Review Comment:
Very useful suggestion! Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
