This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e757345452 Fix example_bigtable system test (#42250)
e757345452 is described below
commit e757345452ea97fcdd307b44eed01d43c104f0b5
Author: olegkachur-e <[email protected]>
AuthorDate: Wed Sep 18 21:25:31 2024 +0200
Fix example_bigtable system test (#42250)
---
.../google/cloud/bigtable/example_bigtable.py | 58 +++++++++++++---------
1 file changed, 34 insertions(+), 24 deletions(-)
diff --git a/tests/system/providers/google/cloud/bigtable/example_bigtable.py
b/tests/system/providers/google/cloud/bigtable/example_bigtable.py
index e99cd62ae5..01f2faff80 100644
--- a/tests/system/providers/google/cloud/bigtable/example_bigtable.py
+++ b/tests/system/providers/google/cloud/bigtable/example_bigtable.py
@@ -67,20 +67,22 @@ ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or
DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "bigtable"
-
-CBT_INSTANCE_ID = f"bigtable-instance-id-{ENV_ID}"
-CBT_INSTANCE_DISPLAY_NAME = "Instance-name"
-CBT_INSTANCE_DISPLAY_NAME_UPDATED = f"{CBT_INSTANCE_DISPLAY_NAME} - updated"
+# CBT instance id str full length should be between [6,33], lowercase only,
"_" symbol is forbidden
+CBT_INSTANCE_ID_BASE = f"inst-id-{str(ENV_ID)[:20]}".lower().replace("_", "-")
+CBT_INSTANCE_ID_1 = f"{CBT_INSTANCE_ID_BASE}-1"
+CBT_INSTANCE_ID_2 = f"{CBT_INSTANCE_ID_BASE}-2"
+CBT_INSTANCE_DISPLAY_NAME = "instance-name"
+CBT_INSTANCE_DISPLAY_NAME_UPDATED = f"{CBT_INSTANCE_DISPLAY_NAME}-updated"
CBT_INSTANCE_TYPE = enums.Instance.Type.DEVELOPMENT
-CBT_INSTANCE_TYPE_PROD = 1
+CBT_INSTANCE_TYPE_PROD = enums.Instance.Type.PRODUCTION
CBT_INSTANCE_LABELS: dict[str, str] = {}
CBT_INSTANCE_LABELS_UPDATED = {"env": "prod"}
-CBT_CLUSTER_ID = f"bigtable-cluster-id-{ENV_ID}"
+CBT_CLUSTER_ID = "bigtable-cluster-id"
CBT_CLUSTER_ZONE = "europe-west1-b"
CBT_CLUSTER_NODES = 3
CBT_CLUSTER_NODES_UPDATED = 5
CBT_CLUSTER_STORAGE_TYPE = enums.StorageType.HDD
-CBT_TABLE_ID = f"bigtable-table-id{ENV_ID}"
+CBT_TABLE_ID = "bigtable-table-id"
CBT_POKE_INTERVAL = 60
@@ -94,7 +96,7 @@ with DAG(
# [START howto_operator_gcp_bigtable_instance_create]
create_instance_task = BigtableCreateInstanceOperator(
project_id=PROJECT_ID,
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
main_cluster_id=CBT_CLUSTER_ID,
main_cluster_zone=CBT_CLUSTER_ZONE,
instance_display_name=CBT_INSTANCE_DISPLAY_NAME,
@@ -104,8 +106,10 @@ with DAG(
cluster_storage_type=CBT_CLUSTER_STORAGE_TYPE, # type:
ignore[arg-type]
task_id="create_instance_task",
)
+ # [END howto_operator_gcp_bigtable_instance_create]
+
create_instance_task2 = BigtableCreateInstanceOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
main_cluster_id=CBT_CLUSTER_ID,
main_cluster_zone=CBT_CLUSTER_ZONE,
instance_display_name=CBT_INSTANCE_DISPLAY_NAME,
@@ -115,23 +119,24 @@ with DAG(
cluster_storage_type=CBT_CLUSTER_STORAGE_TYPE, # type:
ignore[arg-type]
task_id="create_instance_task2",
)
- # [END howto_operator_gcp_bigtable_instance_create]
@task_group()
def create_tables():
# [START howto_operator_gcp_bigtable_table_create]
create_table_task = BigtableCreateTableOperator(
project_id=PROJECT_ID,
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
table_id=CBT_TABLE_ID,
task_id="create_table",
)
+ # [END howto_operator_gcp_bigtable_table_create]
+
create_table_task2 = BigtableCreateTableOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
table_id=CBT_TABLE_ID,
task_id="create_table_task2",
)
- # [END howto_operator_gcp_bigtable_table_create]
+
create_table_task >> create_table_task2
@task_group()
@@ -139,22 +144,22 @@ with DAG(
# [START howto_operator_gcp_bigtable_cluster_update]
cluster_update_task = BigtableUpdateClusterOperator(
project_id=PROJECT_ID,
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
cluster_id=CBT_CLUSTER_ID,
nodes=CBT_CLUSTER_NODES_UPDATED,
task_id="update_cluster_task",
)
+ # [END howto_operator_gcp_bigtable_cluster_update]
cluster_update_task2 = BigtableUpdateClusterOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
cluster_id=CBT_CLUSTER_ID,
nodes=CBT_CLUSTER_NODES_UPDATED,
task_id="update_cluster_task2",
)
- # [END howto_operator_gcp_bigtable_cluster_update]
# [START howto_operator_gcp_bigtable_instance_update]
update_instance_task = BigtableUpdateInstanceOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
instance_display_name=CBT_INSTANCE_DISPLAY_NAME_UPDATED,
instance_type=CBT_INSTANCE_TYPE_PROD,
instance_labels=CBT_INSTANCE_LABELS_UPDATED,
@@ -166,7 +171,7 @@ with DAG(
# [START howto_operator_gcp_bigtable_table_wait_for_replication]
wait_for_table_replication_task = BigtableTableReplicationCompletedSensor(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
table_id=CBT_TABLE_ID,
poke_interval=CBT_POKE_INTERVAL,
timeout=180,
@@ -177,38 +182,43 @@ with DAG(
# [START howto_operator_gcp_bigtable_table_delete]
delete_table_task = BigtableDeleteTableOperator(
project_id=PROJECT_ID,
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
table_id=CBT_TABLE_ID,
task_id="delete_table_task",
)
+ # [END howto_operator_gcp_bigtable_table_delete]
delete_table_task2 = BigtableDeleteTableOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
table_id=CBT_TABLE_ID,
task_id="delete_table_task2",
)
- # [END howto_operator_gcp_bigtable_table_delete]
delete_table_task.trigger_rule = TriggerRule.ALL_DONE
delete_table_task2.trigger_rule = TriggerRule.ALL_DONE
# [START howto_operator_gcp_bigtable_instance_delete]
delete_instance_task = BigtableDeleteInstanceOperator(
project_id=PROJECT_ID,
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_1,
task_id="delete_instance_task",
)
+ # [END howto_operator_gcp_bigtable_instance_delete]
+
delete_instance_task2 = BigtableDeleteInstanceOperator(
- instance_id=CBT_INSTANCE_ID,
+ instance_id=CBT_INSTANCE_ID_2,
task_id="delete_instance_task2",
)
- # [END howto_operator_gcp_bigtable_instance_delete]
+
delete_instance_task.trigger_rule = TriggerRule.ALL_DONE
delete_instance_task2.trigger_rule = TriggerRule.ALL_DONE
(
+ # TEST SETUP
[create_instance_task, create_instance_task2]
+ # TEST BODY
>> create_tables()
>> wait_for_table_replication_task
>> update_clusters_and_instance()
+ # TEST TEARDOWN
>> delete_table_task
>> delete_table_task2
>> [delete_instance_task, delete_instance_task2]
