olayiwola-gbenga commented on code in PR #74664:
URL: https://github.com/apache/airflow/pull/74664#discussion_r4527055211
##########
providers/google/src/airflow/providers/google/firebase/hooks/firestore.py:
##########
@@ -84,9 +91,188 @@ def get_conn(self):
self._conn = build_from_document(non_authorized_conn._rootDesc,
http=http_authorized)
return self._conn
+ def get_operation(self, operation_name: str) -> dict[str, Any]:
+ """
+ Retrieve the current state of a long-running Firestore operation.
+
+ :param operation_name: The resource name of the operation.
+ """
+ service = self.get_conn()
+ return (
+ service.projects()
+ .databases()
+ .operations()
+ .get(name=operation_name)
+ .execute(num_retries=self.num_retries)
+ )
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def list_operations(
+ self, database_id: str = "(default)", project_id: str =
PROVIDE_PROJECT_ID
+ ) -> list[dict[str, Any]]:
+ """
+ List long-running operations on the specified Firestore database.
+
+ :param database_id: The Database ID.
+ :param project_id: Optional, Google Cloud Project project_id where the
database belongs.
+ If set to None or missing, the default project_id from the Google
Cloud connection is used.
+ """
+ service = self.get_conn()
+ name = f"projects/{project_id}/databases/{database_id}"
+ operations_resource = service.projects().databases().operations()
+ all_operations: list[dict[str, Any]] = []
+ page_token: str | None = None
+
+ while True:
+ list_kwargs: dict[str, Any] = {"name": name}
+ if page_token:
+ list_kwargs["pageToken"] = page_token
+ response =
operations_resource.list(**list_kwargs).execute(num_retries=self.num_retries)
+ if not isinstance(response, dict):
+ break
+ operations = response.get("operations", [])
+ if isinstance(operations, list):
+ all_operations.extend(operations)
+ next_page_token = response.get("nextPageToken")
+ if not isinstance(next_page_token, str) or not next_page_token:
+ break
+ page_token = next_page_token
+
+ return all_operations
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def find_matching_export_operation(
+ self,
+ body: dict[str, Any],
+ database_id: str = "(default)",
+ project_id: str = PROVIDE_PROJECT_ID,
+ include_completed: bool = True,
+ ) -> dict[str, Any] | None:
+ """
+ Find an existing running or completed export operation matching the
export request body.
+
+ :param body: The export request body.
+ :param database_id: The Database ID.
+ :param project_id: Optional, Google Cloud Project project_id where the
database belongs.
+ :param include_completed: Whether to match already completed
operations in addition to
+ in-progress operations.
+ """
+ target_uri = body.get("outputUriPrefix")
+ if not target_uri or not isinstance(target_uri, str):
+ return None
+ target_collections = sorted(body.get("collectionIds") or [])
+ target_namespaces = sorted(body.get("namespaceIds") or [])
+
+ completed_match: dict[str, Any] | None = None
+ for op in self.list_operations(database_id=database_id,
project_id=project_id):
+ if not isinstance(op, dict):
+ continue
+ metadata = op.get("metadata")
+ if not isinstance(metadata, dict):
+ metadata = {}
+
+ meta_type = metadata.get("@type", "")
+ if meta_type and not meta_type.endswith("ExportDocumentsMetadata"):
+ continue
+
+ op_state = metadata.get("operationState", "")
+ if op.get("error") or op_state in {"FAILED", "CANCELLED",
"CANCELLING"}:
+ continue
+
+ op_collections = sorted(metadata.get("collectionIds") or [])
+ if target_collections != op_collections:
+ continue
+
+ op_namespaces = sorted(metadata.get("namespaceIds") or [])
+ if target_namespaces != op_namespaces:
+ continue
+
+ response_dict = op.get("response")
+ op_uri = metadata.get("outputUriPrefix") or (
+ response_dict.get("outputUriPrefix") if
isinstance(response_dict, dict) else None
+ )
+ if not isinstance(op_uri, str) or op_uri.rstrip("/") !=
target_uri.rstrip("/"):
+ continue
+
+ is_done = bool(op.get("done"))
+ if not is_done:
+ return op
+ if include_completed and completed_match is None:
+ completed_match = op
+
+ return completed_match
+
+ @GoogleBaseHook.fallback_to_default_project_id
+ def start_export_documents(
+ self,
+ body: dict[str, Any],
+ database_id: str = "(default)",
+ project_id: str = PROVIDE_PROJECT_ID,
+ reattach_on_restart: bool = True,
+ ) -> dict[str, Any]:
+ """
+ Start a Firestore export or return an existing running/completed
operation when reattaching.
+
+ :param body: The request body.
+ :param database_id: The Database ID.
+ :param project_id: Optional, Google Cloud Project project_id where the
database belongs.
+ :param reattach_on_restart: If True, check for an existing in-progress
export operation
+ matching ``body`` before submitting a new export request, and
recover if the export
+ already completed on a prior attempt.
+ """
+ if reattach_on_restart:
+ existing_op = self.find_matching_export_operation(
+ body=body,
+ database_id=database_id,
+ project_id=project_id,
+ include_completed=False,
+ )
+ if existing_op is not None:
+ self.log.info(
+ "Reattaching to in-progress Firestore export operation
%s.",
+ existing_op.get("name"),
+ )
+ return existing_op
+
+ service = self.get_conn()
+ name = f"projects/{project_id}/databases/{database_id}"
+ try:
+ return (
+ service.projects()
+ .databases()
+ .exportDocuments(name=name, body=body)
+ .execute(num_retries=self.num_retries)
+ )
+ except HttpError as err:
+ if reattach_on_restart and err.resp.status in {400, 409}:
Review Comment:
True, I added min_start_time filtering (populated from dag_run.start_date in
the operator) so completed operations on HTTP 400/409 are only accepted if they
started after the current Dag run, and updated the reattach_on_restart
docstrings.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
