This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f09b2201fa Fix Custom Training Job operators to accept results without
managed model (#40685)
f09b2201fa is described below
commit f09b2201fa1d7370cb4ba4d3c80f79acb4f91970
Author: Eugene <[email protected]>
AuthorDate: Sat Jul 13 16:31:53 2024 +0000
Fix Custom Training Job operators to accept results without managed model
(#40685)
---
.../google/cloud/hooks/vertex_ai/custom_job.py | 2 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 106 ++++-------
.../google/cloud/operators/test_vertex_ai.py | 210 +++++++++++++++++++--
.../example_vertex_ai_custom_container.py | 10 +-
.../example_vertex_ai_custom_job_python_package.py | 10 +-
5 files changed, 235 insertions(+), 103 deletions(-)
diff --git a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
index c3139030aa..3a431d0192 100644
--- a/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
+++ b/airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py
@@ -262,7 +262,7 @@ class CustomJobHook(GoogleBaseHook):
@staticmethod
def extract_model_id_from_training_pipeline(training_pipeline: dict[str,
Any]) -> str:
- """Return a unique Model id from a serialized TrainingPipeline
proto."""
+ """Return a unique Model ID from a serialized TrainingPipeline
proto."""
return training_pipeline["model_to_upload"]["name"].rpartition("/")[-1]
@staticmethod
diff --git a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
index d8dc2a0081..7b2964da8e 100644
--- a/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
+++ b/airflow/providers/google/cloud/operators/vertex_ai/custom_job.py
@@ -180,6 +180,40 @@ class
CustomTrainingJobBaseOperator(GoogleCloudBaseOperator):
stacklevel=2,
)
+ def execute_complete(self, context: Context, event: dict[str, Any]) ->
dict[str, Any] | None:
+ if event["status"] == "error":
+ raise AirflowException(event["message"])
+ training_pipeline = event["job"]
+ custom_job_id =
self.hook.extract_custom_job_id_from_training_pipeline(training_pipeline)
+ self.xcom_push(context, key="custom_job_id", value=custom_job_id)
+ try:
+ model = training_pipeline["model_to_upload"]
+ model_id = self.hook.extract_model_id(model)
+ self.xcom_push(context, key="model_id", value=model_id)
+ VertexAIModelLink.persist(context=context, task_instance=self,
model_id=model_id)
+ return model
+ except KeyError:
+ self.log.warning(
+ "It is impossible to get the Model. "
+ "The Training Pipeline did not produce a Managed Model because
it was not "
+ "configured to upload a Model. Please ensure that the
'model_serving_container_image_uri' "
+ "and 'model_display_name' parameters are passed in when
creating a Training Pipeline, "
+ "and check that your training script saves the model to
os.environ['AIP_MODEL_DIR']."
+ )
+ return None
+
+ @cached_property
+ def hook(self) -> CustomJobHook:
+ return CustomJobHook(
+ gcp_conn_id=self.gcp_conn_id,
+ impersonation_chain=self.impersonation_chain,
+ )
+
+ def on_kill(self) -> None:
+ """Act as a callback called when the operator is killed; cancel any
running job."""
+ if self.hook:
+ self.hook.cancel_job()
+
class CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
"""
@@ -565,23 +599,6 @@ class
CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
VertexAITrainingLink.persist(context=context, task_instance=self,
training_id=training_id)
return result
- def on_kill(self) -> None:
- """Act as a callback called when the operator is killed; cancel any
running job."""
- if self.hook:
- self.hook.cancel_job()
-
- def execute_complete(self, context: Context, event: dict[str, Any]) ->
dict[str, Any] | None:
- if event["status"] == "error":
- raise AirflowException(event["message"])
- result = event["job"]
- model_id = self.hook.extract_model_id_from_training_pipeline(result)
- custom_job_id =
self.hook.extract_custom_job_id_from_training_pipeline(result)
- self.xcom_push(context, key="model_id", value=model_id)
- VertexAIModelLink.persist(context=context, task_instance=self,
model_id=model_id)
- # push custom_job_id to xcom so it could be pulled by other tasks
- self.xcom_push(context, key="custom_job_id", value=custom_job_id)
- return result
-
def invoke_defer(self, context: Context) -> None:
custom_container_training_job_obj: CustomContainerTrainingJob =
self.hook.submit_custom_container_training_job(
project_id=self.project_id,
@@ -651,13 +668,6 @@ class
CreateCustomContainerTrainingJobOperator(CustomTrainingJobBaseOperator):
method_name="execute_complete",
)
- @cached_property
- def hook(self) -> CustomJobHook:
- return CustomJobHook(
- gcp_conn_id=self.gcp_conn_id,
- impersonation_chain=self.impersonation_chain,
- )
-
class
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator):
"""
@@ -1042,23 +1052,6 @@ class
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator
VertexAITrainingLink.persist(context=context, task_instance=self,
training_id=training_id)
return result
- def on_kill(self) -> None:
- """Cancel any running job. Callback called when the operator is
killed."""
- if self.hook:
- self.hook.cancel_job()
-
- def execute_complete(self, context: Context, event: dict[str, Any]) ->
dict[str, Any] | None:
- if event["status"] == "error":
- raise AirflowException(event["message"])
- result = event["job"]
- model_id = self.hook.extract_model_id_from_training_pipeline(result)
- custom_job_id =
self.hook.extract_custom_job_id_from_training_pipeline(result)
- self.xcom_push(context, key="model_id", value=model_id)
- VertexAIModelLink.persist(context=context, task_instance=self,
model_id=model_id)
- # push custom_job_id to xcom so it could be pulled by other tasks
- self.xcom_push(context, key="custom_job_id", value=custom_job_id)
- return result
-
def invoke_defer(self, context: Context) -> None:
custom_python_training_job_obj: CustomPythonPackageTrainingJob =
self.hook.submit_custom_python_package_training_job(
project_id=self.project_id,
@@ -1129,13 +1122,6 @@ class
CreateCustomPythonPackageTrainingJobOperator(CustomTrainingJobBaseOperator
method_name="execute_complete",
)
- @cached_property
- def hook(self) -> CustomJobHook:
- return CustomJobHook(
- gcp_conn_id=self.gcp_conn_id,
- impersonation_chain=self.impersonation_chain,
- )
-
class CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
"""
@@ -1525,23 +1511,6 @@ class
CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
VertexAITrainingLink.persist(context=context, task_instance=self,
training_id=training_id)
return result
- def on_kill(self) -> None:
- """Cancel any running job. Callback called when the operator is
killed."""
- if self.hook:
- self.hook.cancel_job()
-
- def execute_complete(self, context: Context, event: dict[str, Any]) ->
dict[str, Any] | None:
- if event["status"] == "error":
- raise AirflowException(event["message"])
- result = event["job"]
- model_id = self.hook.extract_model_id_from_training_pipeline(result)
- custom_job_id =
self.hook.extract_custom_job_id_from_training_pipeline(result)
- self.xcom_push(context, key="model_id", value=model_id)
- VertexAIModelLink.persist(context=context, task_instance=self,
model_id=model_id)
- # push custom_job_id to xcom so it could be pulled by other tasks
- self.xcom_push(context, key="custom_job_id", value=custom_job_id)
- return result
-
def invoke_defer(self, context: Context) -> None:
custom_training_job_obj: CustomTrainingJob =
self.hook.submit_custom_training_job(
project_id=self.project_id,
@@ -1612,13 +1581,6 @@ class
CreateCustomTrainingJobOperator(CustomTrainingJobBaseOperator):
method_name="execute_complete",
)
- @cached_property
- def hook(self) -> CustomJobHook:
- return CustomJobHook(
- gcp_conn_id=self.gcp_conn_id,
- impersonation_chain=self.impersonation_chain,
- )
-
class DeleteCustomTrainingJobOperator(GoogleCloudBaseOperator):
"""
diff --git a/tests/providers/google/cloud/operators/test_vertex_ai.py
b/tests/providers/google/cloud/operators/test_vertex_ai.py
index a30c8f7a3d..e62dd395ac 100644
--- a/tests/providers/google/cloud/operators/test_vertex_ai.py
+++ b/tests/providers/google/cloud/operators/test_vertex_ai.py
@@ -202,6 +202,17 @@ TEST_TEMPLATE_PATH = "test_template_path"
SYNC_DEPRECATION_WARNING = "The 'sync' parameter is deprecated and will be
removed after {}."
+TEST_TRAINING_PIPELINE_DATA = {
+ "model_to_upload": {
+ "name":
"projects/test-project/locations/us-central1/models/test-model",
+ "display_name": "test-model",
+ },
+ "training_task_metadata": {"backingCustomJob": "prefix/test-custom-job"},
+}
+TEST_TRAINING_PIPELINE_DATA_NO_MODEL = {
+ "training_task_metadata": {"backingCustomJob": "prefix/test-custom-job"}
+}
+
class TestVertexAICreateCustomContainerTrainingJobOperator:
@mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -418,9 +429,15 @@ class TestVertexAICreateCustomContainerTrainingJobOperator:
exc.value.trigger, CustomContainerTrainingJobTrigger
), "Trigger is not a CustomContainerTrainingJobTrigger"
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.xcom_push"))
+ @mock.patch(
+
VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook.extract_model_id")
+ )
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook"))
- def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+ def test_execute_complete_success(
+ self, mock_hook, mock_hook_extract_model_id, mock_xcom_push,
mock_link_persist
+ ):
task = CreateCustomContainerTrainingJobOperator(
task_id=TASK_ID,
gcp_conn_id=GCP_CONN_ID,
@@ -443,12 +460,18 @@ class
TestVertexAICreateCustomContainerTrainingJobOperator:
project_id=GCP_PROJECT,
deferrable=True,
)
- expected_result = {}
- mock_hook.return_value.exists.return_value = False
- mock_xcom_push.return_value = None
+ expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+ mock_hook_extract_model_id.return_value = "test-model"
actual_result = task.execute_complete(
- context=None, event={"status": "success", "message": "", "job": {}}
- )
+ context=None,
+ event={
+ "status": "success",
+ "message": "",
+ "job": TEST_TRAINING_PIPELINE_DATA,
+ },
+ )
+ mock_xcom_push.assert_called_with(None, key="model_id",
value="test-model")
+ mock_link_persist.assert_called_once_with(context=None,
task_instance=task, model_id="test-model")
assert actual_result == expected_result
def test_execute_complete_error_status_raises_exception(self):
@@ -477,6 +500,49 @@ class TestVertexAICreateCustomContainerTrainingJobOperator:
with pytest.raises(AirflowException):
task.execute_complete(context=None, event={"status": "error",
"message": "test message"})
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.xcom_push"))
+ @mock.patch(
+
VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook.extract_model_id")
+ )
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomContainerTrainingJobOperator.hook"))
+ def test_execute_complete_no_model_produced(
+ self,
+ mock_hook,
+ hook_extract_model_id,
+ mock_xcom_push,
+ mock_link_persist,
+ ):
+ task = CreateCustomContainerTrainingJobOperator(
+ task_id=TASK_ID,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ staging_bucket=STAGING_BUCKET,
+ display_name=DISPLAY_NAME,
+ args=ARGS,
+ container_uri=CONTAINER_URI,
+ command=COMMAND_2,
+ replica_count=REPLICA_COUNT,
+ machine_type=MACHINE_TYPE,
+ accelerator_type=ACCELERATOR_TYPE,
+ accelerator_count=ACCELERATOR_COUNT,
+ training_fraction_split=TRAINING_FRACTION_SPLIT,
+ validation_fraction_split=VALIDATION_FRACTION_SPLIT,
+ test_fraction_split=TEST_FRACTION_SPLIT,
+ region=GCP_LOCATION,
+ project_id=GCP_PROJECT,
+ deferrable=True,
+ )
+ expected_result = None
+ hook_extract_model_id.return_value = None
+ actual_result = task.execute_complete(
+ context=None,
+ event={"status": "success", "message": "", "job":
TEST_TRAINING_PIPELINE_DATA_NO_MODEL},
+ )
+ mock_xcom_push.assert_called_once()
+ mock_link_persist.assert_not_called()
+ assert actual_result == expected_result
+
class TestVertexAICreateCustomPythonPackageTrainingJobOperator:
@mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -698,9 +764,19 @@ class
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
exc.value.trigger, CustomPythonPackageTrainingJobTrigger
), "Trigger is not a CustomPythonPackageTrainingJobTrigger"
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.xcom_push"))
+ @mock.patch(
+
VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook.extract_model_id")
+ )
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook"))
- def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+ def test_execute_complete_success(
+ self,
+ mock_hook,
+ hook_extract_model_id,
+ mock_xcom_push,
+ mock_link_persist,
+ ):
task = CreateCustomPythonPackageTrainingJobOperator(
task_id=TASK_ID,
gcp_conn_id=GCP_CONN_ID,
@@ -724,12 +800,18 @@ class
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
project_id=GCP_PROJECT,
deferrable=True,
)
- expected_result = {}
- mock_hook.return_value.exists.return_value = False
- mock_xcom_push.return_value = None
+ expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+ hook_extract_model_id.return_value = "test-model"
actual_result = task.execute_complete(
- context=None, event={"status": "success", "message": "", "job": {}}
- )
+ context=None,
+ event={
+ "status": "success",
+ "message": "",
+ "job": TEST_TRAINING_PIPELINE_DATA,
+ },
+ )
+ mock_xcom_push.assert_called_with(None, key="model_id",
value="test-model")
+ mock_link_persist.assert_called_once_with(context=None,
task_instance=task, model_id="test-model")
assert actual_result == expected_result
def test_execute_complete_error_status_raises_exception(self):
@@ -759,6 +841,49 @@ class
TestVertexAICreateCustomPythonPackageTrainingJobOperator:
with pytest.raises(AirflowException):
task.execute_complete(context=None, event={"status": "error",
"message": "test message"})
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.xcom_push"))
+ @mock.patch(
+
VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook.extract_model_id")
+ )
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomPythonPackageTrainingJobOperator.hook"))
+ def test_execute_complete_no_model_produced(
+ self,
+ mock_hook,
+ hook_extract_model_id,
+ mock_xcom_push,
+ mock_link_persist,
+ ):
+ task = CreateCustomPythonPackageTrainingJobOperator(
+ task_id=TASK_ID,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ staging_bucket=STAGING_BUCKET,
+ display_name=DISPLAY_NAME,
+ python_package_gcs_uri=PYTHON_PACKAGE_GCS_URI,
+ python_module_name=PYTHON_MODULE_NAME,
+ container_uri=CONTAINER_URI,
+ args=ARGS,
+ replica_count=REPLICA_COUNT,
+ machine_type=MACHINE_TYPE,
+ accelerator_type=ACCELERATOR_TYPE,
+ accelerator_count=ACCELERATOR_COUNT,
+ training_fraction_split=TRAINING_FRACTION_SPLIT,
+ validation_fraction_split=VALIDATION_FRACTION_SPLIT,
+ test_fraction_split=TEST_FRACTION_SPLIT,
+ region=GCP_LOCATION,
+ project_id=GCP_PROJECT,
+ deferrable=True,
+ )
+ expected_result = None
+ actual_result = task.execute_complete(
+ context=None,
+ event={"status": "success", "message": "", "job":
TEST_TRAINING_PIPELINE_DATA_NO_MODEL},
+ )
+ mock_xcom_push.assert_called_once()
+ mock_link_persist.assert_not_called()
+ assert actual_result == expected_result
+
class TestVertexAICreateCustomTrainingJobOperator:
@mock.patch(VERTEX_AI_PATH.format("custom_job.Dataset"))
@@ -959,9 +1084,17 @@ class TestVertexAICreateCustomTrainingJobOperator:
exc.value.trigger, CustomTrainingJobTrigger
), "Trigger is not a CustomTrainingJobTrigger"
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.xcom_push"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook.extract_model_id"))
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook"))
- def test_execute_complete_success(self, mock_hook, mock_xcom_push):
+ def test_execute_complete_success(
+ self,
+ mock_hook,
+ hook_extract_model_id,
+ mock_xcom_push,
+ mock_link_persist,
+ ):
task = CreateCustomTrainingJobOperator(
task_id=TASK_ID,
gcp_conn_id=GCP_CONN_ID,
@@ -978,12 +1111,18 @@ class TestVertexAICreateCustomTrainingJobOperator:
project_id=GCP_PROJECT,
deferrable=True,
)
- expected_result = {}
- mock_hook.return_value.exists.return_value = False
- mock_xcom_push.return_value = None
+ expected_result = TEST_TRAINING_PIPELINE_DATA["model_to_upload"]
+ hook_extract_model_id.return_value = "test-model"
actual_result = task.execute_complete(
- context=None, event={"status": "success", "message": "", "job": {}}
- )
+ context=None,
+ event={
+ "status": "success",
+ "message": "",
+ "job": TEST_TRAINING_PIPELINE_DATA,
+ },
+ )
+ mock_xcom_push.assert_called_with(None, key="model_id",
value="test-model")
+ mock_link_persist.assert_called_once_with(context=None,
task_instance=task, model_id="test-model")
assert actual_result == expected_result
def test_execute_complete_error_status_raises_exception(self):
@@ -1006,6 +1145,41 @@ class TestVertexAICreateCustomTrainingJobOperator:
with pytest.raises(AirflowException):
task.execute_complete(context=None, event={"status": "error",
"message": "test message"})
+ @mock.patch(VERTEX_AI_LINKS_PATH.format("VertexAIModelLink.persist"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.xcom_push"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook.extract_model_id"))
+
@mock.patch(VERTEX_AI_PATH.format("custom_job.CreateCustomTrainingJobOperator.hook"))
+ def test_execute_complete_no_model_produced(
+ self,
+ mock_hook,
+ hook_extract_model_id,
+ mock_xcom_push,
+ mock_link_persist,
+ ):
+ task = CreateCustomTrainingJobOperator(
+ task_id=TASK_ID,
+ gcp_conn_id=GCP_CONN_ID,
+ impersonation_chain=IMPERSONATION_CHAIN,
+ staging_bucket=STAGING_BUCKET,
+ display_name=DISPLAY_NAME,
+ script_path=PYTHON_PACKAGE,
+ args=PYTHON_PACKAGE_CMDARGS,
+ container_uri=CONTAINER_URI,
+ requirements=[],
+ replica_count=1,
+ region=GCP_LOCATION,
+ project_id=GCP_PROJECT,
+ deferrable=True,
+ )
+ expected_result = None
+ hook_extract_model_id.return_value = None
+ actual_result = task.execute_complete(
+ context=None, event={"status": "success", "message": "", "job": {}}
+ )
+ mock_xcom_push.assert_called_once()
+ mock_link_persist.assert_not_called()
+ assert actual_result == expected_result
+
class TestVertexAIDeleteCustomTrainingJobOperator:
@mock.patch(VERTEX_AI_PATH.format("custom_job.CustomJobHook"))
diff --git
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
index df1a3e5306..ff642ae953 100644
---
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
+++
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_container.py
@@ -17,9 +17,7 @@
# under the License.
-"""
-Example Airflow DAG for Google Vertex AI service testing Custom Jobs
operations.
-"""
+"""Example Airflow DAG for Google Vertex AI service testing Custom Jobs
operations."""
from __future__ import annotations
@@ -48,7 +46,7 @@ from airflow.utils.trigger_rule import TriggerRule
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
-DAG_ID = "example_vertex_ai_custom_job_operations"
+DAG_ID = "vertex_ai_custom_job_operations"
REGION = "us-central1"
CONTAINER_DISPLAY_NAME = f"train-housing-container-{ENV_ID}"
MODEL_DISPLAY_NAME = f"container-housing-model-{ENV_ID}"
@@ -140,13 +138,13 @@ with DAG(
create_custom_container_training_job_deferrable =
CreateCustomContainerTrainingJobOperator(
task_id="custom_container_task_deferrable",
staging_bucket=f"gs://{CUSTOM_CONTAINER_GCS_BUCKET_NAME}",
- display_name=f"{CONTAINER_DISPLAY_NAME}_DEF",
+ display_name=f"{CONTAINER_DISPLAY_NAME}-def",
container_uri=CUSTOM_CONTAINER_URI,
model_serving_container_image_uri=MODEL_SERVING_CONTAINER_URI,
# run params
dataset_id=tabular_dataset_id,
command=["python3", "task.py"],
- model_display_name=f"{MODEL_DISPLAY_NAME}_DEF",
+ model_display_name=f"{MODEL_DISPLAY_NAME}-def",
replica_count=REPLICA_COUNT,
machine_type=MACHINE_TYPE,
accelerator_type=ACCELERATOR_TYPE,
diff --git
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
index 5aad847f84..c9fd96aee9 100644
---
a/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
+++
b/tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_custom_job_python_package.py
@@ -17,9 +17,7 @@
# under the License.
-"""
-Example Airflow DAG for Google Vertex AI service testing Custom Jobs
operations.
-"""
+"""Example Airflow DAG for Google Vertex AI service testing Custom Jobs
operations."""
from __future__ import annotations
@@ -48,7 +46,7 @@ from airflow.utils.trigger_rule import TriggerRule
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
-DAG_ID = "example_vertex_ai_custom_job_operations"
+DAG_ID = "vertex_ai_custom_job_operations"
REGION = "us-central1"
PACKAGE_DISPLAY_NAME = f"train-housing-py-package-{ENV_ID}"
MODEL_DISPLAY_NAME = f"py-package-housing-model-{ENV_ID}"
@@ -143,14 +141,14 @@ with DAG(
create_custom_python_package_training_job_deferrable =
CreateCustomPythonPackageTrainingJobOperator(
task_id="python_package_task_deferrable",
staging_bucket=f"gs://{CUSTOM_PYTHON_GCS_BUCKET_NAME}",
- display_name=f"{PACKAGE_DISPLAY_NAME}_DEF",
+ display_name=f"{PACKAGE_DISPLAY_NAME}-def",
python_package_gcs_uri=PYTHON_PACKAGE_GCS_URI,
python_module_name=PYTHON_MODULE_NAME,
container_uri=CONTAINER_URI,
model_serving_container_image_uri=MODEL_SERVING_CONTAINER_URI,
# run params
dataset_id=tabular_dataset_id,
- model_display_name=f"{MODEL_DISPLAY_NAME}_DEF",
+ model_display_name=f"{MODEL_DISPLAY_NAME}-def",
replica_count=REPLICA_COUNT,
machine_type=MACHINE_TYPE,
accelerator_type=ACCELERATOR_TYPE,