chamikaramj commented on code in PR #41439: URL: https://github.com/apache/beam/pull/41439#discussion_r4608092520
########## examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py: ########## @@ -0,0 +1,423 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Airflow DAG for scheduled Delta Lake to GCP Lakehouse CDC sync. + +Uses ``DataflowStartFlexTemplateOperator`` to run Apache Beam's +``DeltaLakeToLakehouseCdcExample`` Dataflow Flex Template across advancing, +inclusive Delta Lake commit version boundaries (``[startVersion, endVersion]``). + +Delta Commit Version Boundary Safeguards: +1. Inclusive Boundary Arithmetic: Both ``startVersion`` and ``endVersion`` in + ``Managed.DELTA_LAKE_CDC`` are inclusive. Given ``last_synced_version``, the + next run begins strictly at ``start_version = last_synced_version + 1``. +2. Explicit Upper-Bound Pinning: ``end_version`` is resolved from the source + Delta Lake ``_delta_log`` before submitting the Dataflow job so concurrent + writes during job execution cannot create gaps or duplicate commits. +3. No-Op Short-Circuiting: When ``start_version > latest_version``, the DAG + branches cleanly to ``skip_no_new_commits`` without launching Dataflow. +4. CDF Validation & Schema-Evolution Splitting: Verifies that + ``delta.enableChangeDataFeed = true`` holds across the range, and when + ``stop_before_schema_change=True`` caps ``end_version`` at ``V - 1`` if a + schema change occurs at commit ``V > start_version``. +5. Single Active Run & Atomic Watermark Advancement: ``max_active_runs=1`` + prevents concurrent runs from racing on the watermark, and + ``advance_version_watermark`` updates the Airflow Variable only after + ``DataflowStartFlexTemplateOperator(wait_until_finished=True)`` succeeds. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +from airflow import DAG +from airflow.decorators import task +from airflow.hooks.base import BaseHook +from airflow.models import Variable +from airflow.models.param import Param +from airflow.operators.empty import EmptyOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.google.cloud.operators.dataflow import ( + DataflowStartFlexTemplateOperator, +) + +log = logging.getLogger(__name__) + +GCP_CONN_ID = "google_cloud_default" +DEFAULT_WATERMARK_VAR = "delta_to_lakehouse_cdc_last_synced_version" + + +def compute_delta_version_boundary( + last_synced_version: int, + latest_version: int, + get_version_info: Callable[[int], tuple[bool, str]], + max_versions_per_run: int | None = None, + stop_before_schema_change: bool = True, + delta_table_uri: str = "", + watermark_var: str = DEFAULT_WATERMARK_VAR, +) -> dict[str, Any]: + """Computes the inclusive ``[start_version, end_version]`` CDC commit range. + + Args: + last_synced_version: Highest Delta commit version already migrated to the + target Lakehouse table (for example, the initial snapshot version or the + commit that enabled Change Data Feed). + latest_version: Current latest commit version of the source Delta table. + get_version_info: Callback taking a Delta commit version ``v`` and + returning ``(cdf_enabled, schema_json)`` at version ``v``. + max_versions_per_run: Optional upper cap on the number of commits processed + per Dataflow run. + stop_before_schema_change: When ``True``, if a schema change occurs at + commit ``V > start_version`` inside the candidate range, caps + ``end_version`` at ``V - 1`` so pre-schema-change commits migrate first. + delta_table_uri: Source Delta table URI (used in log/error messages). + watermark_var: Airflow Variable name (used in log/error messages). + + Returns: + A dictionary with ``has_new_commits``, ``last_synced_version``, + ``latest_version``, ``start_version``, and ``end_version``. + """ + # Both startVersion and endVersion in Managed.DELTA_LAKE_CDC are INCLUSIVE. + start_version = last_synced_version + 1 + + if start_version > latest_version: + return { + "has_new_commits": False, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": None, + "end_version": None, + } + + candidate_end_version = latest_version + if max_versions_per_run is not None and int(max_versions_per_run) > 0: + candidate_end_version = min( + latest_version, start_version + int(max_versions_per_run) - 1 + ) + + start_cdf_enabled, start_schema_json = get_version_info(start_version) + if not start_cdf_enabled: + raise ValueError( + f"Delta Lake table '{delta_table_uri}' does not have " + f"'delta.enableChangeDataFeed = true' at start_version={start_version}. " + f"Advance '{watermark_var}' to at least the commit where Change Data " + f"Feed was enabled." + ) + + if last_synced_version >= 0: + _, prev_schema_json = get_version_info(last_synced_version) + if start_schema_json != prev_schema_json: + log.warning( + "Commit start_version=%d has a different Delta schema than " + "last_synced_version=%d. Ensure the target Lakehouse Iceberg table " + "schema has been altered accordingly before migrating commit %d.", + start_version, + last_synced_version, + start_version, + ) + + end_version = candidate_end_version + for ver in range(start_version + 1, candidate_end_version + 1): + ver_cdf_enabled, ver_schema_json = get_version_info(ver) + if not ver_cdf_enabled: + raise ValueError( + f"Change Data Feed was disabled at commit version {ver} on " + f"'{delta_table_uri}'." + ) + if stop_before_schema_change and ver_schema_json != start_schema_json: + end_version = ver - 1 + log.warning( + "Detected Delta schema change at commit version %d. Capping current " + "CDC batch end_version=%d so pre-schema-change commits [%d, %d] " + "synchronize cleanly first.", + ver, + end_version, + start_version, + end_version, + ) + break + + return { + "has_new_commits": True, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": start_version, + "end_version": end_version, + } + + +def _build_delta_storage_options(gcp_conn_id: str) -> dict[str, str]: + """Builds delta-rs GCS storage_options from Airflow's GCP connection or ADC.""" + try: + conn = BaseHook.get_connection(gcp_conn_id) + extra = conn.extra_dejson or {} + key_path = ( + extra.get("key_path") + or extra.get("extra__google_cloud_platform__key_path") + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + ) + if key_path and os.path.exists(key_path): + return {"google_service_account_path": key_path} + except Exception as exc: + log.debug( + "Could not read key_path from Airflow connection '%s': %s", + gcp_conn_id, + exc, + ) + + env_key = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if env_key and os.path.exists(env_key): + return {"google_service_account_path": env_key} + + # Cloud Composer / Workload Identity / ADC fallback using OAuth2 token. + gcs_hook = GCSHook(gcp_conn_id=gcp_conn_id) + creds = gcs_hook.get_credentials() + import google.auth.transport.requests + + creds.refresh(google.auth.transport.requests.Request()) + return {"google_storage_token": creds.token} + + +with DAG( + dag_id="delta_lake_to_lakehouse_cdc_sync", + description=( + "Incrementally synchronizes Delta Lake Change Data Feed (CDC) commits " + "to a GCP Lakehouse (Iceberg) table via a Dataflow Flex Template." + ), + schedule=timedelta(minutes=30), + start_date=datetime(2026, 1, 1), + catchup=False, + max_active_runs=1, # Prevent overlapping runs from sharing a watermark. + default_args={ + "owner": "data-engineering", + "retries": 1, Review Comment: Good catch. Changed retries to "0". This combined with the sink_id change should make it deterministic. ########## examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py: ########## @@ -0,0 +1,423 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Airflow DAG for scheduled Delta Lake to GCP Lakehouse CDC sync. + +Uses ``DataflowStartFlexTemplateOperator`` to run Apache Beam's +``DeltaLakeToLakehouseCdcExample`` Dataflow Flex Template across advancing, +inclusive Delta Lake commit version boundaries (``[startVersion, endVersion]``). + +Delta Commit Version Boundary Safeguards: +1. Inclusive Boundary Arithmetic: Both ``startVersion`` and ``endVersion`` in + ``Managed.DELTA_LAKE_CDC`` are inclusive. Given ``last_synced_version``, the + next run begins strictly at ``start_version = last_synced_version + 1``. +2. Explicit Upper-Bound Pinning: ``end_version`` is resolved from the source + Delta Lake ``_delta_log`` before submitting the Dataflow job so concurrent + writes during job execution cannot create gaps or duplicate commits. +3. No-Op Short-Circuiting: When ``start_version > latest_version``, the DAG + branches cleanly to ``skip_no_new_commits`` without launching Dataflow. +4. CDF Validation & Schema-Evolution Splitting: Verifies that + ``delta.enableChangeDataFeed = true`` holds across the range, and when + ``stop_before_schema_change=True`` caps ``end_version`` at ``V - 1`` if a + schema change occurs at commit ``V > start_version``. +5. Single Active Run & Atomic Watermark Advancement: ``max_active_runs=1`` + prevents concurrent runs from racing on the watermark, and + ``advance_version_watermark`` updates the Airflow Variable only after + ``DataflowStartFlexTemplateOperator(wait_until_finished=True)`` succeeds. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +from airflow import DAG +from airflow.decorators import task +from airflow.hooks.base import BaseHook +from airflow.models import Variable +from airflow.models.param import Param +from airflow.operators.empty import EmptyOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.google.cloud.operators.dataflow import ( + DataflowStartFlexTemplateOperator, +) + +log = logging.getLogger(__name__) + +GCP_CONN_ID = "google_cloud_default" +DEFAULT_WATERMARK_VAR = "delta_to_lakehouse_cdc_last_synced_version" + + +def compute_delta_version_boundary( + last_synced_version: int, + latest_version: int, + get_version_info: Callable[[int], tuple[bool, str]], + max_versions_per_run: int | None = None, + stop_before_schema_change: bool = True, + delta_table_uri: str = "", + watermark_var: str = DEFAULT_WATERMARK_VAR, +) -> dict[str, Any]: + """Computes the inclusive ``[start_version, end_version]`` CDC commit range. + + Args: + last_synced_version: Highest Delta commit version already migrated to the + target Lakehouse table (for example, the initial snapshot version or the + commit that enabled Change Data Feed). + latest_version: Current latest commit version of the source Delta table. + get_version_info: Callback taking a Delta commit version ``v`` and + returning ``(cdf_enabled, schema_json)`` at version ``v``. + max_versions_per_run: Optional upper cap on the number of commits processed + per Dataflow run. + stop_before_schema_change: When ``True``, if a schema change occurs at + commit ``V > start_version`` inside the candidate range, caps + ``end_version`` at ``V - 1`` so pre-schema-change commits migrate first. + delta_table_uri: Source Delta table URI (used in log/error messages). + watermark_var: Airflow Variable name (used in log/error messages). + + Returns: + A dictionary with ``has_new_commits``, ``last_synced_version``, + ``latest_version``, ``start_version``, and ``end_version``. + """ + # Both startVersion and endVersion in Managed.DELTA_LAKE_CDC are INCLUSIVE. + start_version = last_synced_version + 1 + + if start_version > latest_version: + return { + "has_new_commits": False, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": None, + "end_version": None, + } + + candidate_end_version = latest_version + if max_versions_per_run is not None and int(max_versions_per_run) > 0: + candidate_end_version = min( + latest_version, start_version + int(max_versions_per_run) - 1 + ) + + start_cdf_enabled, start_schema_json = get_version_info(start_version) + if not start_cdf_enabled: + raise ValueError( + f"Delta Lake table '{delta_table_uri}' does not have " + f"'delta.enableChangeDataFeed = true' at start_version={start_version}. " + f"Advance '{watermark_var}' to at least the commit where Change Data " + f"Feed was enabled." + ) + + if last_synced_version >= 0: + _, prev_schema_json = get_version_info(last_synced_version) + if start_schema_json != prev_schema_json: + log.warning( + "Commit start_version=%d has a different Delta schema than " + "last_synced_version=%d. Ensure the target Lakehouse Iceberg table " + "schema has been altered accordingly before migrating commit %d.", + start_version, + last_synced_version, + start_version, + ) + + end_version = candidate_end_version + for ver in range(start_version + 1, candidate_end_version + 1): + ver_cdf_enabled, ver_schema_json = get_version_info(ver) + if not ver_cdf_enabled: + raise ValueError( + f"Change Data Feed was disabled at commit version {ver} on " + f"'{delta_table_uri}'." + ) + if stop_before_schema_change and ver_schema_json != start_schema_json: + end_version = ver - 1 + log.warning( + "Detected Delta schema change at commit version %d. Capping current " + "CDC batch end_version=%d so pre-schema-change commits [%d, %d] " + "synchronize cleanly first.", + ver, + end_version, + start_version, + end_version, + ) + break + + return { + "has_new_commits": True, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": start_version, + "end_version": end_version, + } + + +def _build_delta_storage_options(gcp_conn_id: str) -> dict[str, str]: + """Builds delta-rs GCS storage_options from Airflow's GCP connection or ADC.""" + try: + conn = BaseHook.get_connection(gcp_conn_id) + extra = conn.extra_dejson or {} + key_path = ( + extra.get("key_path") + or extra.get("extra__google_cloud_platform__key_path") + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + ) + if key_path and os.path.exists(key_path): + return {"google_service_account_path": key_path} + except Exception as exc: + log.debug( + "Could not read key_path from Airflow connection '%s': %s", + gcp_conn_id, + exc, + ) + + env_key = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if env_key and os.path.exists(env_key): + return {"google_service_account_path": env_key} + + # Cloud Composer / Workload Identity / ADC fallback using OAuth2 token. + gcs_hook = GCSHook(gcp_conn_id=gcp_conn_id) + creds = gcs_hook.get_credentials() + import google.auth.transport.requests + + creds.refresh(google.auth.transport.requests.Request()) + return {"google_storage_token": creds.token} Review Comment: Yeah, AI review was right that this is not needed. It was silently being ignored by my runs. Updated the code now. I did confirm that this runs end-to-end by generating a Flex template and running with a local Airflow deployment BTW. Added details to the PR description. ########## examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py: ########## @@ -0,0 +1,423 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Airflow DAG for scheduled Delta Lake to GCP Lakehouse CDC sync. + +Uses ``DataflowStartFlexTemplateOperator`` to run Apache Beam's +``DeltaLakeToLakehouseCdcExample`` Dataflow Flex Template across advancing, +inclusive Delta Lake commit version boundaries (``[startVersion, endVersion]``). + +Delta Commit Version Boundary Safeguards: +1. Inclusive Boundary Arithmetic: Both ``startVersion`` and ``endVersion`` in + ``Managed.DELTA_LAKE_CDC`` are inclusive. Given ``last_synced_version``, the + next run begins strictly at ``start_version = last_synced_version + 1``. +2. Explicit Upper-Bound Pinning: ``end_version`` is resolved from the source + Delta Lake ``_delta_log`` before submitting the Dataflow job so concurrent + writes during job execution cannot create gaps or duplicate commits. +3. No-Op Short-Circuiting: When ``start_version > latest_version``, the DAG + branches cleanly to ``skip_no_new_commits`` without launching Dataflow. +4. CDF Validation & Schema-Evolution Splitting: Verifies that + ``delta.enableChangeDataFeed = true`` holds across the range, and when + ``stop_before_schema_change=True`` caps ``end_version`` at ``V - 1`` if a + schema change occurs at commit ``V > start_version``. +5. Single Active Run & Atomic Watermark Advancement: ``max_active_runs=1`` + prevents concurrent runs from racing on the watermark, and + ``advance_version_watermark`` updates the Airflow Variable only after + ``DataflowStartFlexTemplateOperator(wait_until_finished=True)`` succeeds. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +from airflow import DAG +from airflow.decorators import task +from airflow.hooks.base import BaseHook +from airflow.models import Variable +from airflow.models.param import Param +from airflow.operators.empty import EmptyOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.google.cloud.operators.dataflow import ( + DataflowStartFlexTemplateOperator, +) + +log = logging.getLogger(__name__) + +GCP_CONN_ID = "google_cloud_default" +DEFAULT_WATERMARK_VAR = "delta_to_lakehouse_cdc_last_synced_version" + + +def compute_delta_version_boundary( + last_synced_version: int, + latest_version: int, + get_version_info: Callable[[int], tuple[bool, str]], + max_versions_per_run: int | None = None, + stop_before_schema_change: bool = True, + delta_table_uri: str = "", + watermark_var: str = DEFAULT_WATERMARK_VAR, +) -> dict[str, Any]: + """Computes the inclusive ``[start_version, end_version]`` CDC commit range. + + Args: + last_synced_version: Highest Delta commit version already migrated to the + target Lakehouse table (for example, the initial snapshot version or the + commit that enabled Change Data Feed). + latest_version: Current latest commit version of the source Delta table. + get_version_info: Callback taking a Delta commit version ``v`` and + returning ``(cdf_enabled, schema_json)`` at version ``v``. + max_versions_per_run: Optional upper cap on the number of commits processed + per Dataflow run. + stop_before_schema_change: When ``True``, if a schema change occurs at + commit ``V > start_version`` inside the candidate range, caps + ``end_version`` at ``V - 1`` so pre-schema-change commits migrate first. + delta_table_uri: Source Delta table URI (used in log/error messages). + watermark_var: Airflow Variable name (used in log/error messages). + + Returns: + A dictionary with ``has_new_commits``, ``last_synced_version``, + ``latest_version``, ``start_version``, and ``end_version``. + """ + # Both startVersion and endVersion in Managed.DELTA_LAKE_CDC are INCLUSIVE. + start_version = last_synced_version + 1 + + if start_version > latest_version: + return { + "has_new_commits": False, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": None, + "end_version": None, + } + + candidate_end_version = latest_version + if max_versions_per_run is not None and int(max_versions_per_run) > 0: + candidate_end_version = min( + latest_version, start_version + int(max_versions_per_run) - 1 + ) + + start_cdf_enabled, start_schema_json = get_version_info(start_version) + if not start_cdf_enabled: + raise ValueError( + f"Delta Lake table '{delta_table_uri}' does not have " + f"'delta.enableChangeDataFeed = true' at start_version={start_version}. " + f"Advance '{watermark_var}' to at least the commit where Change Data " + f"Feed was enabled." + ) + + if last_synced_version >= 0: + _, prev_schema_json = get_version_info(last_synced_version) + if start_schema_json != prev_schema_json: + log.warning( + "Commit start_version=%d has a different Delta schema than " + "last_synced_version=%d. Ensure the target Lakehouse Iceberg table " + "schema has been altered accordingly before migrating commit %d.", + start_version, + last_synced_version, + start_version, + ) + + end_version = candidate_end_version + for ver in range(start_version + 1, candidate_end_version + 1): + ver_cdf_enabled, ver_schema_json = get_version_info(ver) + if not ver_cdf_enabled: + raise ValueError( + f"Change Data Feed was disabled at commit version {ver} on " + f"'{delta_table_uri}'." + ) + if stop_before_schema_change and ver_schema_json != start_schema_json: + end_version = ver - 1 + log.warning( + "Detected Delta schema change at commit version %d. Capping current " + "CDC batch end_version=%d so pre-schema-change commits [%d, %d] " + "synchronize cleanly first.", + ver, + end_version, + start_version, + end_version, + ) + break + + return { + "has_new_commits": True, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": start_version, + "end_version": end_version, + } + + +def _build_delta_storage_options(gcp_conn_id: str) -> dict[str, str]: + """Builds delta-rs GCS storage_options from Airflow's GCP connection or ADC.""" + try: + conn = BaseHook.get_connection(gcp_conn_id) + extra = conn.extra_dejson or {} + key_path = ( + extra.get("key_path") + or extra.get("extra__google_cloud_platform__key_path") + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + ) + if key_path and os.path.exists(key_path): + return {"google_service_account_path": key_path} + except Exception as exc: + log.debug( + "Could not read key_path from Airflow connection '%s': %s", + gcp_conn_id, + exc, + ) + + env_key = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if env_key and os.path.exists(env_key): + return {"google_service_account_path": env_key} + + # Cloud Composer / Workload Identity / ADC fallback using OAuth2 token. + gcs_hook = GCSHook(gcp_conn_id=gcp_conn_id) + creds = gcs_hook.get_credentials() + import google.auth.transport.requests + + creds.refresh(google.auth.transport.requests.Request()) + return {"google_storage_token": creds.token} + + +with DAG( + dag_id="delta_lake_to_lakehouse_cdc_sync", + description=( + "Incrementally synchronizes Delta Lake Change Data Feed (CDC) commits " + "to a GCP Lakehouse (Iceberg) table via a Dataflow Flex Template." + ), + schedule=timedelta(minutes=30), + start_date=datetime(2026, 1, 1), + catchup=False, + max_active_runs=1, # Prevent overlapping runs from sharing a watermark. + default_args={ + "owner": "data-engineering", + "retries": 1, + "retry_delay": timedelta(minutes=2), + }, + params={ + "project_id": Param("my-gcp-project", type="string"), + "location": Param("us-central1", type="string"), + "template_gcs_path": Param( + "gs://my-temp-bucket/templates/delta-lake-to-lakehouse-cdc.json", + type="string", + ), + "temp_location": Param("gs://my-temp-bucket/temp", type="string"), + "staging_location": Param("gs://my-temp-bucket/staging", type="string"), + "delta_table": Param( + "gs://my-delta-lake-bucket/delta_lake/employee_data/", + type="string", + ), + "lakehouse_table": Param( + "my-gcp-project.my-lakehouse-warehouse-bucket.my_namespace.my_iceberg_table", + type="string", + ), + "equality_columns": Param("employee_id", type="string"), + "use_portable_runner": Param("true", type="string"), + "watermark_variable": Param(DEFAULT_WATERMARK_VAR, type="string"), + "initial_synced_version": Param( + 1, + type="integer", + description=( + "Baseline Delta commit version already covered by the initial " + "snapshot / CDF enablement when the Airflow watermark Variable " + "has not been set yet. The first CDC run starts at " + "initial_synced_version + 1." + ), Review Comment: Done. ########## examples/java/iceberg/airflow/README.md: ########## @@ -0,0 +1,224 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Automating Delta Lake to GCP Lakehouse CDC with Dataflow Flex Templates and Apache Airflow + +This directory provides everything needed to package Apache Beam's +[`DeltaLakeToLakehouseCdcExample`](../src/main/java/org/apache/beam/examples/iceberg/DeltaLakeToLakehouseCdcExample.java) +as a **Cloud Dataflow Flex Template** and schedule repeated Change Data Capture +(CDC) synchronizations with **Apache Airflow** (or **Cloud Composer**) using +`DataflowStartFlexTemplateOperator`. + +## Contents + +- [`flex_template_metadata.json`](flex_template_metadata.json): Dataflow Flex + Template parameter metadata for `DeltaLakeToLakehouseCdcExample`. +- [`delta_lake_to_lakehouse_cdc_dag.py`](delta_lake_to_lakehouse_cdc_dag.py): + Airflow DAG that tracks the Delta Lake commit watermark, resolves inclusive + `[startVersion, endVersion]` boundaries, invokes + `DataflowStartFlexTemplateOperator`, and advances the watermark atomically on + success. + +For the initial full-table snapshot migration before starting incremental CDC, +see [`examples/java/iceberg/sql/delta_to_iceberg.sql`](../sql/delta_to_iceberg.sql) +(and [`examples/java/iceberg/sql/delta_to_iceberg_cdc.sql`](../sql/delta_to_iceberg_cdc.sql) +for one-off Beam SQL CDC runs). + +--- + +## Prerequisites + +1. **Java 17+**: Both Delta Kernel and Apache Iceberg require Java 17 or later. + If using SDKMAN: + ```bash + sdk use java 17.0.15-tem + ``` +2. **Source Delta Lake table on GCS** with Change Data Feed (CDF) enabled: + ```sql + ALTER TABLE delta.`gs://my-delta-lake-bucket/delta_lake/employee_data` + SET TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true'); + ``` +3. **Target GCP Lakehouse (Iceberg V2) table** initialized with the baseline + snapshot pinned to a known commit version (for example, version `1`, right + after enabling CDF; subsequent CDC runs then start at version `2`). +4. **GCP Resources & IAM**: + - Enabled APIs: Dataflow, Cloud Build, Artifact Registry, Cloud Storage, + BigLake. + - An Artifact Registry Docker repository (e.g., + `us-central1-docker.pkg.dev/my-gcp-project/my-repo`). + - Service account permissions for GCS buckets, Lakehouse catalog + (`roles/biglake.admin`), Dataflow (`roles/dataflow.admin` / + `roles/dataflow.worker`), and Artifact Registry. + +--- + +## Step 1: Build the Flex Template Uber-JAR + +From the root of the Apache Beam repository, run the dedicated Gradle task to +produce a shaded uber-JAR containing `DeltaLakeToLakehouseCdcExample`, +`Managed.DELTA_LAKE_CDC`, `Managed.ICEBERG`, `DataflowRunner`, and merged +`META-INF/services` entries: + +```bash +./gradlew :examples:java:iceberg:deltaLakeToLakehouseCdcFlexTemplateJar +``` + +This creates: +```text +examples/java/iceberg/build/libs/delta-lake-to-lakehouse-cdc-flex-template-<beam-version>-bundled.jar +``` + +> **Note for standalone Maven projects:** If you build outside the Beam +> repository using Maven, configure `maven-shade-plugin` with +> `ServicesResourceTransformer` and align all `com.google.cloud.bigdataoss` +> artifacts (`gcs-connector`, `gcsio`, `util-hadoop`, `util`) to the same +> version in `<dependencyManagement>` to avoid runtime GCS Hadoop connector +> mismatches in the Flex Template launcher. + +--- + +## Step 2: Build and Publish the Dataflow Flex Template + +Use `gcloud dataflow flex-template build` to build the Java 17 Flex Template +container image, push it to Artifact Registry (or GCR), and upload the template +specification JSON to Cloud Storage: + +```bash +export PROJECT_ID="my-gcp-project" +export REGION="us-central1" +export REPO_NAME="my-docker-repo" +export BUCKET="my-temp-bucket" +export JAR_PATH=$(ls examples/java/iceberg/build/libs/delta-lake-to-lakehouse-cdc-flex-template-*-bundled.jar | head -n 1) + +gcloud dataflow flex-template build \ + "gs://${BUCKET}/templates/delta-lake-to-lakehouse-cdc.json" \ + --project="${PROJECT_ID}" \ + --image-gcr-path="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO_NAME}/delta-lake-to-lakehouse-cdc:latest" \ + --sdk-language="JAVA" \ + --flex-template-base-image="JAVA17" \ + --metadata-file="examples/java/iceberg/airflow/flex_template_metadata.json" \ + --jar="${JAR_PATH}" \ + --env="FLEX_TEMPLATE_JAVA_MAIN_CLASS=org.apache.beam.examples.iceberg.DeltaLakeToLakehouseCdcExample" +``` + +--- + +## Step 3: Deploy and Configure the Airflow DAG + +### Option A: Cloud Composer (Recommended for Production) + +1. Install the `deltalake` PyPI package on your Cloud Composer environment + (`gcloud composer environments update ... --update-pypi-packages-from-file` + or via the Cloud Console). +2. Update the default `params` in `delta_lake_to_lakehouse_cdc_dag.py` (or pass + them via DAG run config / Airflow Variables) for your `project_id`, + `location`, `template_gcs_path`, `delta_table`, `lakehouse_table`, and + `equality_columns`. +3. Upload the DAG to Composer: + ```bash + gcloud composer environments storage dags import \ + --environment="my-composer-env" \ + --location="us-central1" \ + --source="examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py" + ``` +4. Set the initial commit watermark variable to the baseline commit version + already covered by your initial snapshot: + ```bash + gcloud composer environments run my-composer-env \ + --location="us-central1" \ + variables -- set delta_to_lakehouse_cdc_last_synced_version 1 + ``` + +### Option B: Local Apache Airflow (Development & Testing) + +```bash +python3.11 -m venv airflow_venv +AIRFLOW_VERSION=2.10.5 +CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.11.txt" + +./airflow_venv/bin/pip install --upgrade pip +./airflow_venv/bin/pip install \ + "apache-airflow==${AIRFLOW_VERSION}" \ + "apache-airflow-providers-google" \ + --constraint "${CONSTRAINT_URL}" +# Airflow's Dataflow hook imports apache-beam provider helpers; --no-deps avoids +# needing local Python apache-beam when launching Java Flex Templates remotely. +./airflow_venv/bin/pip install "apache-airflow-providers-apache-beam==6.0.0" --no-deps +./airflow_venv/bin/pip install deltalake + +export AIRFLOW_HOME="$(pwd)/airflow_local" +export AIRFLOW__CORE__DAGS_FOLDER="$(pwd)/examples/java/iceberg/airflow" +export AIRFLOW__CORE__LOAD_EXAMPLES=False + +./airflow_venv/bin/airflow db migrate +./airflow_venv/bin/airflow connections add google_cloud_default \ + --conn-type google_cloud_platform \ + --conn-extra '{"project": "my-gcp-project", "key_path": "/path/to/service-account-key.json"}' + +./airflow_venv/bin/airflow variables set delta_to_lakehouse_cdc_last_synced_version 1 + +# Run a single DAG execution with runtime parameter overrides: +./airflow_venv/bin/airflow dags test delta_lake_to_lakehouse_cdc_sync \ + --conf '{ + "project_id": "my-gcp-project", + "location": "us-central1", + "template_gcs_path": "gs://my-temp-bucket/templates/delta-lake-to-lakehouse-cdc.json", + "temp_location": "gs://my-temp-bucket/temp", + "staging_location": "gs://my-temp-bucket/staging", + "delta_table": "gs://my-delta-lake-bucket/delta_lake/employee_data/", + "lakehouse_table": "my-gcp-project.my-lakehouse-warehouse-bucket.my_namespace.my_iceberg_table", + "equality_columns": "employee_id" + }' +``` + +--- + +## How Delta Version ID Boundaries Are Handled + +`delta_lake_to_lakehouse_cdc_dag.py` protects against duplicate inserts, missed +commits, and schema-evolution mismatches with five safeguards: Review Comment: Done. ########## examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py: ########## @@ -0,0 +1,423 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Airflow DAG for scheduled Delta Lake to GCP Lakehouse CDC sync. + +Uses ``DataflowStartFlexTemplateOperator`` to run Apache Beam's +``DeltaLakeToLakehouseCdcExample`` Dataflow Flex Template across advancing, +inclusive Delta Lake commit version boundaries (``[startVersion, endVersion]``). + +Delta Commit Version Boundary Safeguards: +1. Inclusive Boundary Arithmetic: Both ``startVersion`` and ``endVersion`` in + ``Managed.DELTA_LAKE_CDC`` are inclusive. Given ``last_synced_version``, the + next run begins strictly at ``start_version = last_synced_version + 1``. +2. Explicit Upper-Bound Pinning: ``end_version`` is resolved from the source + Delta Lake ``_delta_log`` before submitting the Dataflow job so concurrent + writes during job execution cannot create gaps or duplicate commits. +3. No-Op Short-Circuiting: When ``start_version > latest_version``, the DAG + branches cleanly to ``skip_no_new_commits`` without launching Dataflow. +4. CDF Validation & Schema-Evolution Splitting: Verifies that + ``delta.enableChangeDataFeed = true`` holds across the range, and when + ``stop_before_schema_change=True`` caps ``end_version`` at ``V - 1`` if a + schema change occurs at commit ``V > start_version``. +5. Single Active Run & Atomic Watermark Advancement: ``max_active_runs=1`` + prevents concurrent runs from racing on the watermark, and + ``advance_version_watermark`` updates the Airflow Variable only after + ``DataflowStartFlexTemplateOperator(wait_until_finished=True)`` succeeds. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +from airflow import DAG +from airflow.decorators import task +from airflow.hooks.base import BaseHook +from airflow.models import Variable +from airflow.models.param import Param +from airflow.operators.empty import EmptyOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.google.cloud.operators.dataflow import ( + DataflowStartFlexTemplateOperator, +) + +log = logging.getLogger(__name__) + +GCP_CONN_ID = "google_cloud_default" +DEFAULT_WATERMARK_VAR = "delta_to_lakehouse_cdc_last_synced_version" + + +def compute_delta_version_boundary( + last_synced_version: int, + latest_version: int, + get_version_info: Callable[[int], tuple[bool, str]], + max_versions_per_run: int | None = None, + stop_before_schema_change: bool = True, + delta_table_uri: str = "", + watermark_var: str = DEFAULT_WATERMARK_VAR, +) -> dict[str, Any]: + """Computes the inclusive ``[start_version, end_version]`` CDC commit range. + + Args: + last_synced_version: Highest Delta commit version already migrated to the + target Lakehouse table (for example, the initial snapshot version or the + commit that enabled Change Data Feed). + latest_version: Current latest commit version of the source Delta table. + get_version_info: Callback taking a Delta commit version ``v`` and + returning ``(cdf_enabled, schema_json)`` at version ``v``. + max_versions_per_run: Optional upper cap on the number of commits processed + per Dataflow run. + stop_before_schema_change: When ``True``, if a schema change occurs at + commit ``V > start_version`` inside the candidate range, caps + ``end_version`` at ``V - 1`` so pre-schema-change commits migrate first. + delta_table_uri: Source Delta table URI (used in log/error messages). + watermark_var: Airflow Variable name (used in log/error messages). + + Returns: + A dictionary with ``has_new_commits``, ``last_synced_version``, + ``latest_version``, ``start_version``, and ``end_version``. + """ + # Both startVersion and endVersion in Managed.DELTA_LAKE_CDC are INCLUSIVE. + start_version = last_synced_version + 1 + + if start_version > latest_version: + return { + "has_new_commits": False, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": None, + "end_version": None, + } + + candidate_end_version = latest_version + if max_versions_per_run is not None and int(max_versions_per_run) > 0: + candidate_end_version = min( + latest_version, start_version + int(max_versions_per_run) - 1 + ) + + start_cdf_enabled, start_schema_json = get_version_info(start_version) + if not start_cdf_enabled: + raise ValueError( + f"Delta Lake table '{delta_table_uri}' does not have " + f"'delta.enableChangeDataFeed = true' at start_version={start_version}. " + f"Advance '{watermark_var}' to at least the commit where Change Data " + f"Feed was enabled." + ) + + if last_synced_version >= 0: + _, prev_schema_json = get_version_info(last_synced_version) + if start_schema_json != prev_schema_json: + log.warning( + "Commit start_version=%d has a different Delta schema than " + "last_synced_version=%d. Ensure the target Lakehouse Iceberg table " + "schema has been altered accordingly before migrating commit %d.", + start_version, + last_synced_version, + start_version, Review Comment: Done. ########## examples/java/iceberg/airflow/delta_lake_to_lakehouse_cdc_dag.py: ########## @@ -0,0 +1,423 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Apache Airflow DAG for scheduled Delta Lake to GCP Lakehouse CDC sync. + +Uses ``DataflowStartFlexTemplateOperator`` to run Apache Beam's +``DeltaLakeToLakehouseCdcExample`` Dataflow Flex Template across advancing, +inclusive Delta Lake commit version boundaries (``[startVersion, endVersion]``). + +Delta Commit Version Boundary Safeguards: +1. Inclusive Boundary Arithmetic: Both ``startVersion`` and ``endVersion`` in + ``Managed.DELTA_LAKE_CDC`` are inclusive. Given ``last_synced_version``, the + next run begins strictly at ``start_version = last_synced_version + 1``. +2. Explicit Upper-Bound Pinning: ``end_version`` is resolved from the source + Delta Lake ``_delta_log`` before submitting the Dataflow job so concurrent + writes during job execution cannot create gaps or duplicate commits. +3. No-Op Short-Circuiting: When ``start_version > latest_version``, the DAG + branches cleanly to ``skip_no_new_commits`` without launching Dataflow. +4. CDF Validation & Schema-Evolution Splitting: Verifies that + ``delta.enableChangeDataFeed = true`` holds across the range, and when + ``stop_before_schema_change=True`` caps ``end_version`` at ``V - 1`` if a + schema change occurs at commit ``V > start_version``. +5. Single Active Run & Atomic Watermark Advancement: ``max_active_runs=1`` + prevents concurrent runs from racing on the watermark, and + ``advance_version_watermark`` updates the Airflow Variable only after + ``DataflowStartFlexTemplateOperator(wait_until_finished=True)`` succeeds. +""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Callable + +from airflow import DAG +from airflow.decorators import task +from airflow.hooks.base import BaseHook +from airflow.models import Variable +from airflow.models.param import Param +from airflow.operators.empty import EmptyOperator +from airflow.providers.google.cloud.hooks.gcs import GCSHook +from airflow.providers.google.cloud.operators.dataflow import ( + DataflowStartFlexTemplateOperator, +) + +log = logging.getLogger(__name__) + +GCP_CONN_ID = "google_cloud_default" +DEFAULT_WATERMARK_VAR = "delta_to_lakehouse_cdc_last_synced_version" + + +def compute_delta_version_boundary( + last_synced_version: int, + latest_version: int, + get_version_info: Callable[[int], tuple[bool, str]], + max_versions_per_run: int | None = None, + stop_before_schema_change: bool = True, + delta_table_uri: str = "", + watermark_var: str = DEFAULT_WATERMARK_VAR, +) -> dict[str, Any]: + """Computes the inclusive ``[start_version, end_version]`` CDC commit range. + + Args: + last_synced_version: Highest Delta commit version already migrated to the + target Lakehouse table (for example, the initial snapshot version or the + commit that enabled Change Data Feed). + latest_version: Current latest commit version of the source Delta table. + get_version_info: Callback taking a Delta commit version ``v`` and + returning ``(cdf_enabled, schema_json)`` at version ``v``. + max_versions_per_run: Optional upper cap on the number of commits processed + per Dataflow run. + stop_before_schema_change: When ``True``, if a schema change occurs at + commit ``V > start_version`` inside the candidate range, caps + ``end_version`` at ``V - 1`` so pre-schema-change commits migrate first. + delta_table_uri: Source Delta table URI (used in log/error messages). + watermark_var: Airflow Variable name (used in log/error messages). + + Returns: + A dictionary with ``has_new_commits``, ``last_synced_version``, + ``latest_version``, ``start_version``, and ``end_version``. + """ + # Both startVersion and endVersion in Managed.DELTA_LAKE_CDC are INCLUSIVE. + start_version = last_synced_version + 1 + + if start_version > latest_version: + return { + "has_new_commits": False, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": None, + "end_version": None, + } + + candidate_end_version = latest_version + if max_versions_per_run is not None and int(max_versions_per_run) > 0: + candidate_end_version = min( + latest_version, start_version + int(max_versions_per_run) - 1 + ) + + start_cdf_enabled, start_schema_json = get_version_info(start_version) + if not start_cdf_enabled: + raise ValueError( + f"Delta Lake table '{delta_table_uri}' does not have " + f"'delta.enableChangeDataFeed = true' at start_version={start_version}. " + f"Advance '{watermark_var}' to at least the commit where Change Data " + f"Feed was enabled." + ) + + if last_synced_version >= 0: + _, prev_schema_json = get_version_info(last_synced_version) + if start_schema_json != prev_schema_json: + log.warning( + "Commit start_version=%d has a different Delta schema than " + "last_synced_version=%d. Ensure the target Lakehouse Iceberg table " + "schema has been altered accordingly before migrating commit %d.", + start_version, + last_synced_version, + start_version, + ) + + end_version = candidate_end_version + for ver in range(start_version + 1, candidate_end_version + 1): + ver_cdf_enabled, ver_schema_json = get_version_info(ver) + if not ver_cdf_enabled: + raise ValueError( + f"Change Data Feed was disabled at commit version {ver} on " + f"'{delta_table_uri}'." + ) + if stop_before_schema_change and ver_schema_json != start_schema_json: + end_version = ver - 1 + log.warning( + "Detected Delta schema change at commit version %d. Capping current " + "CDC batch end_version=%d so pre-schema-change commits [%d, %d] " + "synchronize cleanly first.", + ver, + end_version, + start_version, + end_version, + ) + break + + return { + "has_new_commits": True, + "last_synced_version": last_synced_version, + "latest_version": latest_version, + "start_version": start_version, + "end_version": end_version, + } + + +def _build_delta_storage_options(gcp_conn_id: str) -> dict[str, str]: + """Builds delta-rs GCS storage_options from Airflow's GCP connection or ADC.""" + try: + conn = BaseHook.get_connection(gcp_conn_id) + extra = conn.extra_dejson or {} + key_path = ( + extra.get("key_path") + or extra.get("extra__google_cloud_platform__key_path") + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + ) + if key_path and os.path.exists(key_path): + return {"google_service_account_path": key_path} + except Exception as exc: + log.debug( + "Could not read key_path from Airflow connection '%s': %s", + gcp_conn_id, + exc, + ) + + env_key = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + if env_key and os.path.exists(env_key): + return {"google_service_account_path": env_key} + + # Cloud Composer / Workload Identity / ADC fallback using OAuth2 token. + gcs_hook = GCSHook(gcp_conn_id=gcp_conn_id) + creds = gcs_hook.get_credentials() + import google.auth.transport.requests + + creds.refresh(google.auth.transport.requests.Request()) + return {"google_storage_token": creds.token} + + +with DAG( + dag_id="delta_lake_to_lakehouse_cdc_sync", + description=( + "Incrementally synchronizes Delta Lake Change Data Feed (CDC) commits " + "to a GCP Lakehouse (Iceberg) table via a Dataflow Flex Template." + ), + schedule=timedelta(minutes=30), + start_date=datetime(2026, 1, 1), + catchup=False, + max_active_runs=1, # Prevent overlapping runs from sharing a watermark. + default_args={ + "owner": "data-engineering", + "retries": 1, + "retry_delay": timedelta(minutes=2), + }, + params={ + "project_id": Param("my-gcp-project", type="string"), + "location": Param("us-central1", type="string"), + "template_gcs_path": Param( + "gs://my-temp-bucket/templates/delta-lake-to-lakehouse-cdc.json", + type="string", + ), + "temp_location": Param("gs://my-temp-bucket/temp", type="string"), + "staging_location": Param("gs://my-temp-bucket/staging", type="string"), + "delta_table": Param( + "gs://my-delta-lake-bucket/delta_lake/employee_data/", + type="string", + ), + "lakehouse_table": Param( + "my-gcp-project.my-lakehouse-warehouse-bucket.my_namespace.my_iceberg_table", + type="string", + ), + "equality_columns": Param("employee_id", type="string"), + "use_portable_runner": Param("true", type="string"), + "watermark_variable": Param(DEFAULT_WATERMARK_VAR, type="string"), + "initial_synced_version": Param( + 1, + type="integer", + description=( + "Baseline Delta commit version already covered by the initial " + "snapshot / CDF enablement when the Airflow watermark Variable " + "has not been set yet. The first CDC run starts at " + "initial_synced_version + 1." + ), + ), + "max_versions_per_run": Param( + None, + type=["null", "integer"], + description=( + "Optional cap on the number of Delta commits processed in a " + "single Dataflow run. Leave null to sync up to the latest " + "commit (or schema-change boundary)." + ), + ), + "stop_before_schema_change": Param( + True, + type="boolean", + description=( + "If true and a Delta schema change occurs at commit " + "V > start_version, caps end_version at V - 1 so " + "pre-schema-change commits migrate cleanly first." + ), + ), + }, + tags=["dataflow", "delta-lake", "lakehouse", "iceberg", "cdc"], +) as dag: + + @task(task_id="resolve_version_boundary") + def resolve_version_boundary(**context: Any) -> dict[str, Any]: + """Resolves the exact inclusive [start_version, end_version] boundary.""" + # Imported lazily inside the task so Airflow DAG parsing remains lightweight. + from deltalake import DeltaTable + + params = context["params"] + ti = context["ti"] + delta_table_uri: str = params["delta_table"] + watermark_var: str = params["watermark_variable"] + initial_synced_version: int = int(params["initial_synced_version"]) + max_versions_per_run: int | None = params.get("max_versions_per_run") + stop_before_schema_change: bool = bool( + params.get("stop_before_schema_change", True) + ) + + raw_watermark = Variable.get(watermark_var, default_var=None) + if raw_watermark is None: + last_synced_version = initial_synced_version + log.info( + "Airflow Variable '%s' is not set yet; using " + "initial_synced_version=%d.", + watermark_var, + last_synced_version, + ) + else: + last_synced_version = int(raw_watermark) + log.info( + "Loaded last_synced_version=%d from Airflow Variable '%s'.", + last_synced_version, + watermark_var, + ) + + storage_options = _build_delta_storage_options(GCP_CONN_ID) + dt_latest = DeltaTable(delta_table_uri, storage_options=storage_options) Review Comment: AI reply :) ` I checked this against deltalake (1.6.6) and without_files=True is now deprecated and ignored (it emits DeprecationWarning: The 'without_files' parameter is deprecated and ignored. Table loading materializes active-file metadata.). Instead of passing the deprecated flag, I made version inspection leaner by caching latest_version's metadata/schema from the initial DeltaTable load and reusing that single DeltaTable instance via dt.load_as_version(version) instead of constructing a new DeltaTable + GCS client per commit version (in local testing against GCS, this cut a 4-version check from ~4.2s to ~2.6s with zero deprecation warnings). ` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
