This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new f09b2201fa Fix Custom Training Job operators to accept results without 
managed model (#40685)
f09b2201fa is described below

commit f09b2201fa1d7370cb4ba4d3c80f79acb4f91970
Author: Eugene <[email protected]>
AuthorDate: Sat Jul 13 16:31:53 2024 +0000

    Fix Custom Training Job operators to accept results without managed model 
(#40685)
---
 .../google/cloud/hooks/vertex_ai/custom_job.py     |   2 +-
 .../google/cloud/operators/vertex_ai/custom_job.py | 106 ++++-------
 .../google/cloud/operators/test_vertex_ai.py       | 210 +++++++++++++++++++--
 .../example_vertex_ai_custom_container.py          |  10 +-
 .../example_vertex_ai_custom_job_python_package.py |  10 +-
 5 files changed, 235 insertions(+), 103 deletions(-)

diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py 
b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
index c3139030aa..3a431d0192 100644
--- a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
+++ b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
@@ -262,7 +262,7 @@ class CustomJobHook(GoogleBaseHook):
 
     @staticmethod
     def extract_model_id_from_training_pipeline(training_pipeline: dict[str, 
Any]) -> str:
-        """Return a unique Model id from a serialized TrainingPipeline 
proto."""
+        """Return a unique Model ID from a serialized TrainingPipeline 
proto."""
         return training_pipeline["model_to_upload"]["name"].rpartition("/")[-1]
 
     @staticmethod
diff --git a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py 
b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
index d8dc2a0081..7b2964da8e 100644
--- a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
+++ b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
@@ -180,6 +180,40 @@ class 
CustomTrainingJobBaseOperator(GoogleCloudBaseOperator):
             stacklevel=2,
         )
 
+    def execute_complete(self, context: Context, event: dict[str, Any]) -> 
dict[str, Any] | None:
+        if event["status"] == "error":
+            raise AirflowException(event["message"])
+        training_pipeline = event["job"]
+        custom_job_id = 
self.hook.extract_custom_job_id_from_training_pipeline(training_pipeline)
+        self.xcom_push(context, key="custom_job_id", value=custom_job_id)
+        try:
+            model = training_pipeline["model_to_upload"]
+            model_id = self.hook.extract_model_id(model)
+            self.xcom_push(context, key="model_id", value=model_id)
+            VertexAIModelLink.persist(context=context, task_instance=self, 
model_id=model_id)
+            return model
+        except KeyError:
+            self.log.warning(
+                "It is impossible to get the Model. "
+                "The Training Pipeline did not produce a Managed Model because 
it was not "
+                "configured to upload a Model. Please ensure that the 
'model_serving_container_image_uri' "
+                "and 'model_display_name' parameters are passed in when 
creating a Training Pipeline, "
+                "and check that your training script saves the model to 
os.environ['AIP_MODEL_DIR']."
+            )
+            return None
+
+    @cached_property
+    def hook(self) -> CustomJobHook:
+        return CustomJobHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+    def on_kill(self) -> None:
+        """Act as a callback called when the operator is killed; cancel any 
running job."""
+        if self.hook:
+            self.hook.cancel_job()
+
 
 class CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
     """
@@ -565,23 +599,6 @@ class 
CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
         VertexAITrainingLink.persist(context=context, task_instance=self, 
training_id=training_id)
         return result
 
-    def on_kill(self) -> None:
-        """Act as a callback called when the operator is killed; cancel any 
running job."""
-        if self.hook:
-            self.hook.cancel_job()
-
-    def execute_complete(self, context: Context, event: dict[str, Any]) -> 
dict[str, Any] | None:
-        if event["status"] == "error":
-            raise AirflowException(event["message"])
-        result = event["job"]
-        model_id = self.hook.extract_model_id_from_training_pipeline(result)
-        custom_job_id = 
self.hook.extract_custom_job_id_from_training_pipeline(result)
-        self.xcom_push(context, key="model_id", value=model_id)
-        VertexAIModelLink.persist(context=context, task_instance=self, 
model_id=model_id)
-        # push custom_job_id to xcom so it could be pulled by other tasks
-        self.xcom_push(context, key="custom_job_id", value=custom_job_id)
-        return result
-
     def invoke_defer(self, context: Context) -> None:
         custom_container_training_job_obj: CustomContainerTrainingJob = 
self.hook.submit_custom_container_training_job(
             project_id=self.project_id,
@@ -651,13 +668,6 @@ class 
CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
             method_name="execute_complete",
         )
 
-    @cached_property
-    def hook(self) -> CustomJobHook:
-        return CustomJobHook(
-            gcp_conn_id=self.gcp_conn_id,
-            impersonation_chain=self.impersonation_chain,
-        )
-
 
 class 
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator):
     """
@@ -1042,23 +1052,6 @@ class 
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator
         VertexAITrainingLink.persist(context=context, task_instance=self, 
training_id=training_id)
         return result
 
-    def on_kill(self) -> None:
-        """Cancel any running job. Callback called when the operator is 
killed."""
-        if self.hook:
-            self.hook.cancel_job()
-
-    def execute_complete(self, context: Context, event: dict[str, Any]) -> 
dict[str, Any] | None:
-        if event["status"] == "error":
-            raise AirflowException(event["message"])
-        result = event["job"]
-        model_id = self.hook.extract_model_id_from_training_pipeline(result)
-        custom_job_id = 
self.hook.extract_custom_job_id_from_training_pipeline(result)
-        self.xcom_push(context, key="model_id", value=model_id)
-        VertexAIModelLink.persist(context=context, task_instance=self, 
model_id=model_id)
-        # push custom_job_id to xcom so it could be pulled by other tasks
-        self.xcom_push(context, key="custom_job_id", value=custom_job_id)
-        return result
-
     def invoke_defer(self, context: Context) -> None:
         custom_python_training_job_obj: CustomPythonPackageTrainingJob = 
self.hook.submit_custom_python_package_training_job(
             project_id=self.project_id,
@@ -1129,13 +1122,6 @@ class 
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator
             method_name="execute_complete",
         )
 
-    @cached_property
-    def hook(self) -> CustomJobHook:
-        return CustomJobHook(
-            gcp_conn_id=self.gcp_conn_id,
-            impersonation_chain=self.impersonation_chain,
-        )
-
 
 class CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
     """
@@ -1525,23 +1511,6 @@ class 
CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
         VertexAITrainingLink.persist(context=context, task_instance=self, 
training_id=training_id)
         return result
 
-    def on_kill(self) -> None:
-        """Cancel any running job. Callback called when the operator is 
killed."""
-        if self.hook:
-            self.hook.cancel_job()
-
-    def execute_complete(self, context: Context, event: dict[str, Any]) -> 
dict[str, Any] | None:
-        if event["status"] == "error":
-            raise AirflowException(event["message"])
-        result = event["job"]
-        model_id = self.hook.extract_model_id_from_training_pipeline(result)
-        custom_job_id = 
self.hook.extract_custom_job_id_from_training_pipeline(result)
-        self.xcom_push(context, key="model_id", value=model_id)
-        VertexAIModelLink.persist(context=context, task_instance=self, 
model_id=model_id)
-        # push custom_job_id to xcom so it could be pulled by other tasks
-        self.xcom_push(context, key="custom_job_id", value=custom_job_id)
-        return result
-
     def invoke_defer(self, context: Context) -> None:
         custom_training_job_obj: CustomTrainingJob = 
self.hook.submit_custom_training_job(
             project_id=self.project_id,
@@ -1612,13 +1581,6 @@ class 
CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
             method_name="execute_complete",
         )
 
-    @cached_property
-    def hook(self) -> CustomJobHook:
-        return CustomJobHook(
-            gcp_conn_id=self.gcp_conn_id,
-            impersonation_chain=self.impersonation_chain,
-        )
-
 
 class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
     """
diff --git a/tests/providers/google/cloud/operators/test_vertex_ai.py 
b/tests/providers/google/cloud/operators/test_vertex_ai.py
index a30c8f7a3d..e62dd395ac 100644
--- a/tests/providers/google/cloud/operators/test_vertex_ai.py
+++ b/tests/providers/google/cloud/operators/test_vertex_ai.py
@@ -202,6 +202,17 @@ TEST_TEMPLATE_PATH = "test_template_path"
 
 SYNC_DEPRECATION_WARNING = "The 'sync' parameter is deprecated and will be 
removed after {}."
 
+TEST_TRAINING_PIPELINE_DATA = {
+    "model_to_upload": {
+        "name": 
"projects/test-project/locations/us-central1/models/test-model",
+        "display_name": "test-model",
+    },
+    "training_task_metadata": {"backingCustomJob": "prefix/test-custom-job"},
+}
+TEST_TRAINING_PIPELINE_DATA_NO_MODEL = {
+    "training_task_metadata": {"backingCustomJob": "prefix/test-custom-job"}
+}
+
 
 class TestVertexAICreateCustomContainerTrainingJobOperator:
     @mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -418,9 +429,15 @@ class TestVertexAICreateCustomContainerTrainingJobOperator:
             exc.value.trigger, CustomContainerTrainingJobTrigger
         ), "Trigger is not a CustomContainerTrainingJobTrigger"
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.xcom_push"))
+    @mock.patch(
+        
VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook.extract_model_id")
+    )
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook"))
-    def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+    def test_execute_complete_success(
+        self, mock_hook, mock_hook_extract_model_id, mock_xcom_push, 
mock_link_persist
+    ):
         task = CreateCustomContainerTrainingJobOperator(
             task_id=TASK_ID,
             gcp_conn_id=GCP_CONN_ID,
@@ -443,12 +460,18 @@ class 
TestVertexAICreateCustomContainerTrainingJobOperator:
             project_id=GCP_PROJECT,
             deferrable=True,
         )
-        expected_result = {}
-        mock_hook.return_value.exists.return_value = False
-        mock_xcom_push.return_value = None
+        expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+        mock_hook_extract_model_id.return_value = "test-model"
         actual_result = task.execute_complete(
-            context=None, event={"status": "success", "message": "", "job": {}}
-        )
+            context=None,
+            event={
+                "status": "success",
+                "message": "",
+                "job": TEST_TRAINING_PIPELINE_DATA,
+            },
+        )
+        mock_xcom_push.assert_called_with(None, key="model_id", 
value="test-model")
+        mock_link_persist.assert_called_once_with(context=None, 
task_instance=task, model_id="test-model")
         assert actual_result == expected_result
 
     def test_execute_complete_error_status_raises_exception(self):
@@ -477,6 +500,49 @@ class TestVertexAICreateCustomContainerTrainingJobOperator:
         with pytest.raises(AirflowException):
             task.execute_complete(context=None, event={"status": "error", 
"message": "test message"})
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.xcom_push"))
+    @mock.patch(
+        
VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook.extract_model_id")
+    )
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook"))
+    def test_execute_complete_no_model_produced(
+        self,
+        mock_hook,
+        hook_extract_model_id,
+        mock_xcom_push,
+        mock_link_persist,
+    ):
+        task = CreateCustomContainerTrainingJobOperator(
+            task_id=TASK_ID,
+            gcp_conn_id=GCP_CONN_ID,
+            impersonation_chain=IMPERSONATION_CHAIN,
+            staging_bucket=STAGING_BUCKET,
+            display_name=DISPLAY_NAME,
+            args=ARGS,
+            container_uri=CONTAINER_URI,
+            command=COMMAND_2,
+            replica_count=REPLICA_COUNT,
+            machine_type=MACHINE_TYPE,
+            accelerator_type=ACCELERATOR_TYPE,
+            accelerator_count=ACCELERATOR_COUNT,
+            training_fraction_split=TRAINING_FRACTION_SPLIT,
+            validation_fraction_split=VALIDATION_FRACTION_SPLIT,
+            test_fraction_split=TEST_FRACTION_SPLIT,
+            region=GCP_LOCATION,
+            project_id=GCP_PROJECT,
+            deferrable=True,
+        )
+        expected_result = None
+        hook_extract_model_id.return_value = None
+        actual_result = task.execute_complete(
+            context=None,
+            event={"status": "success", "message": "", "job": 
TEST_TRAINING_PIPELINE_DATA_NO_MODEL},
+        )
+        mock_xcom_push.assert_called_once()
+        mock_link_persist.assert_not_called()
+        assert actual_result == expected_result
+
 
 class TestVertexAICreateCustomPythonPackageTrainingJobOperator:
     @mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -698,9 +764,19 @@ class 
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
             exc.value.trigger, CustomPythonPackageTrainingJobTrigger
         ), "Trigger is not a CustomPythonPackageTrainingJobTrigger"
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.xcom_push"))
+    @mock.patch(
+        
VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook.extract_model_id")
+    )
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook"))
-    def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+    def test_execute_complete_success(
+        self,
+        mock_hook,
+        hook_extract_model_id,
+        mock_xcom_push,
+        mock_link_persist,
+    ):
         task = CreateCustomPythonPackageTrainingJobOperator(
             task_id=TASK_ID,
             gcp_conn_id=GCP_CONN_ID,
@@ -724,12 +800,18 @@ class 
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
             project_id=GCP_PROJECT,
             deferrable=True,
         )
-        expected_result = {}
-        mock_hook.return_value.exists.return_value = False
-        mock_xcom_push.return_value = None
+        expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+        hook_extract_model_id.return_value = "test-model"
         actual_result = task.execute_complete(
-            context=None, event={"status": "success", "message": "", "job": {}}
-        )
+            context=None,
+            event={
+                "status": "success",
+                "message": "",
+                "job": TEST_TRAINING_PIPELINE_DATA,
+            },
+        )
+        mock_xcom_push.assert_called_with(None, key="model_id", 
value="test-model")
+        mock_link_persist.assert_called_once_with(context=None, 
task_instance=task, model_id="test-model")
         assert actual_result == expected_result
 
     def test_execute_complete_error_status_raises_exception(self):
@@ -759,6 +841,49 @@ class 
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
         with pytest.raises(AirflowException):
             task.execute_complete(context=None, event={"status": "error", 
"message": "test message"})
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.xcom_push"))
+    @mock.patch(
+        
VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook.extract_model_id")
+    )
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook"))
+    def test_execute_complete_no_model_produced(
+        self,
+        mock_hook,
+        hook_extract_model_id,
+        mock_xcom_push,
+        mock_link_persist,
+    ):
+        task = CreateCustomPythonPackageTrainingJobOperator(
+            task_id=TASK_ID,
+            gcp_conn_id=GCP_CONN_ID,
+            impersonation_chain=IMPERSONATION_CHAIN,
+            staging_bucket=STAGING_BUCKET,
+            display_name=DISPLAY_NAME,
+            python_package_gcs_uri=PYTHON_PACKAGE_GCS_URI,
+            python_module_name=PYTHON_MODULE_NAME,
+            container_uri=CONTAINER_URI,
+            args=ARGS,
+            replica_count=REPLICA_COUNT,
+            machine_type=MACHINE_TYPE,
+            accelerator_type=ACCELERATOR_TYPE,
+            accelerator_count=ACCELERATOR_COUNT,
+            training_fraction_split=TRAINING_FRACTION_SPLIT,
+            validation_fraction_split=VALIDATION_FRACTION_SPLIT,
+            test_fraction_split=TEST_FRACTION_SPLIT,
+            region=GCP_LOCATION,
+            project_id=GCP_PROJECT,
+            deferrable=True,
+        )
+        expected_result = None
+        actual_result = task.execute_complete(
+            context=None,
+            event={"status": "success", "message": "", "job": 
TEST_TRAINING_PIPELINE_DATA_NO_MODEL},
+        )
+        mock_xcom_push.assert_called_once()
+        mock_link_persist.assert_not_called()
+        assert actual_result == expected_result
+
 
 class TestVertexAICreateCustomTrainingJobOperator:
     @mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -959,9 +1084,17 @@ class TestVertexAICreateCustomTrainingJobOperator:
             exc.value.trigger, CustomTrainingJobTrigger
         ), "Trigger is not a CustomTrainingJobTrigger"
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.xcom_push"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook.extract_model_id"))
     
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook"))
-    def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+    def test_execute_complete_success(
+        self,
+        mock_hook,
+        hook_extract_model_id,
+        mock_xcom_push,
+        mock_link_persist,
+    ):
         task = CreateCustomTrainingJobOperator(
             task_id=TASK_ID,
             gcp_conn_id=GCP_CONN_ID,
@@ -978,12 +1111,18 @@ class TestVertexAICreateCustomTrainingJobOperator:
             project_id=GCP_PROJECT,
             deferrable=True,
         )
-        expected_result = {}
-        mock_hook.return_value.exists.return_value = False
-        mock_xcom_push.return_value = None
+        expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+        hook_extract_model_id.return_value = "test-model"
         actual_result = task.execute_complete(
-            context=None, event={"status": "success", "message": "", "job": {}}
-        )
+            context=None,
+            event={
+                "status": "success",
+                "message": "",
+                "job": TEST_TRAINING_PIPELINE_DATA,
+            },
+        )
+        mock_xcom_push.assert_called_with(None, key="model_id", 
value="test-model")
+        mock_link_persist.assert_called_once_with(context=None, 
task_instance=task, model_id="test-model")
         assert actual_result == expected_result
 
     def test_execute_complete_error_status_raises_exception(self):
@@ -1006,6 +1145,41 @@ class TestVertexAICreateCustomTrainingJobOperator:
         with pytest.raises(AirflowException):
             task.execute_complete(context=None, event={"status": "error", 
"message": "test message"})
 
+    @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.xcom_push"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook.extract_model_id"))
+    
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook"))
+    def test_execute_complete_no_model_produced(
+        self,
+        mock_hook,
+        hook_extract_model_id,
+        mock_xcom_push,
+        mock_link_persist,
+    ):
+        task = CreateCustomTrainingJobOperator(
+            task_id=TASK_ID,
+            gcp_conn_id=GCP_CONN_ID,
+            impersonation_chain=IMPERSONATION_CHAIN,
+            staging_bucket=STAGING_BUCKET,
+            display_name=DISPLAY_NAME,
+            script_path=PYTHON_PACKAGE,
+            args=PYTHON_PACKAGE_CMDARGS,
+            container_uri=CONTAINER_URI,
+            requirements=[],
+            replica_count=1,
+            region=GCP_LOCATION,
+            project_id=GCP_PROJECT,
+            deferrable=True,
+        )
+        expected_result = None
+        hook_extract_model_id.return_value = None
+        actual_result = task.execute_complete(
+            context=None, event={"status": "success", "message": "", "job": {}}
+        )
+        mock_xcom_push.assert_called_once()
+        mock_link_persist.assert_not_called()
+        assert actual_result == expected_result
+
 
 class TestVertexAIDeleteCustomTrainingJobOperator:
     @mock.patch(VERTEX_AI_PATH.format("custom_job.CustomJobHook"))
diff --git 
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
 
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
index df1a3e5306..ff642ae953 100644
--- 
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
+++ 
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
@@ -17,9 +17,7 @@
 # under the License.
 
 
-"""
-Example Airflow DAG for Google Vertex AI service testing Custom Jobs 
operations.
-"""
+"""Example Airflow DAG for Google Vertex AI service testing Custom Jobs 
operations."""
 
 from __future__ import annotations
 
@@ -48,7 +46,7 @@ from airflow.utils.trigger_rule import TriggerRule
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
-DAG_ID = "example_vertex_ai_custom_job_operations"
+DAG_ID = "vertex_ai_custom_job_operations"
 REGION = "us-central1"
 CONTAINER_DISPLAY_NAME = f"train-housing-container-{ENV_ID}"
 MODEL_DISPLAY_NAME = f"container-housing-model-{ENV_ID}"
@@ -140,13 +138,13 @@ with DAG(
     create_custom_container_training_job_deferrable = 
CreateCustomContainerTrainingJobOperator(
         task_id="custom_container_task_deferrable",
         staging_bucket=f"gs://{CUSTOM_CONTAINER_GCS_BUCKET_NAME}",
-        display_name=f"{CONTAINER_DISPLAY_NAME}_DEF",
+        display_name=f"{CONTAINER_DISPLAY_NAME}-def",
         container_uri=CUSTOM_CONTAINER_URI,
         model_serving_container_image_uri=MODEL_SERVING_CONTAINER_URI,
         # run params
         dataset_id=tabular_dataset_id,
         command=["python3", "task.py"],
-        model_display_name=f"{MODEL_DISPLAY_NAME}_DEF",
+        model_display_name=f"{MODEL_DISPLAY_NAME}-def",
         replica_count=REPLICA_COUNT,
         machine_type=MACHINE_TYPE,
         accelerator_type=ACCELERATOR_TYPE,
diff --git 
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
 
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
index 5aad847f84..c9fd96aee9 100644
--- 
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
+++ 
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
@@ -17,9 +17,7 @@
 # under the License.
 
 
-"""
-Example Airflow DAG for Google Vertex AI service testing Custom Jobs 
operations.
-"""
+"""Example Airflow DAG for Google Vertex AI service testing Custom Jobs 
operations."""
 
 from __future__ import annotations
 
@@ -48,7 +46,7 @@ from airflow.utils.trigger_rule import TriggerRule
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
-DAG_ID = "example_vertex_ai_custom_job_operations"
+DAG_ID = "vertex_ai_custom_job_operations"
 REGION = "us-central1"
 PACKAGE_DISPLAY_NAME = f"train-housing-py-package-{ENV_ID}"
 MODEL_DISPLAY_NAME = f"py-package-housing-model-{ENV_ID}"
@@ -143,14 +141,14 @@ with DAG(
     create_custom_python_package_training_job_deferrable = 
CreateCustomPythonPackageTrainingJobOperator(
         task_id="python_package_task_deferrable",
         staging_bucket=f"gs://{CUSTOM_PYTHON_GCS_BUCKET_NAME}",
-        display_name=f"{PACKAGE_DISPLAY_NAME}_DEF",
+        display_name=f"{PACKAGE_DISPLAY_NAME}-def",
         python_package_gcs_uri=PYTHON_PACKAGE_GCS_URI,
         python_module_name=PYTHON_MODULE_NAME,
         container_uri=CONTAINER_URI,
         model_serving_container_image_uri=MODEL_SERVING_CONTAINER_URI,
         # run params
         dataset_id=tabular_dataset_id,
-        model_display_name=f"{MODEL_DISPLAY_NAME}_DEF",
+        model_display_name=f"{MODEL_DISPLAY_NAME}-def",
         replica_count=REPLICA_COUNT,
         machine_type=MACHINE_TYPE,
         accelerator_type=ACCELERATOR_TYPE,

Reply via email to