grundprinzip commented on code in PR #39695:
URL: https://github.com/apache/spark/pull/39695#discussion_r1083643160
##########
python/pyspark/sql/connect/client.py:
##########
@@ -531,12 +560,16 @@ def _analyze(self, plan: pb2.Plan, explain_mode: str =
"extended") -> AnalyzeRes
req.explain.explain_mode = pb2.Explain.ExplainMode.CODEGEN
else: # formatted
req.explain.explain_mode = pb2.Explain.ExplainMode.FORMATTED
-
try:
- resp = self._stub.AnalyzePlan(req,
metadata=self._builder.metadata())
- if resp.client_id != self._session_id:
- raise SparkConnectException("Received incorrect session
identifier for request.")
- return AnalyzeResult.fromProto(resp)
+ for attempt in Retrying(SparkConnectClient.retry_exception,
**self._retry_policy):
+ with attempt:
+ resp = self._stub.AnalyzePlan(req,
metadata=self._builder.metadata())
+ if resp.client_id != self._session_id:
+ raise SparkConnectException(
+ "Received incorrect session identifier for
request."
Review Comment:
Done.
##########
python/pyspark/sql/connect/client.py:
##########
@@ -567,54 +602,48 @@ def _execute_and_fetch(
logger.info("ExecuteAndFetch")
m: Optional[pb2.ExecutePlanResponse.Metrics] = None
-
batches: List[pa.RecordBatch] = []
try:
- for b in self._stub.ExecutePlan(req,
metadata=self._builder.metadata()):
- if b.client_id != self._session_id:
- raise SparkConnectException(
- "Received incorrect session identifier for request."
- )
- if b.metrics is not None:
- logger.debug("Received metric batch.")
- m = b.metrics
- if b.HasField("arrow_batch"):
- logger.debug(
- f"Received arrow batch rows={b.arrow_batch.row_count} "
- f"size={len(b.arrow_batch.data)}"
- )
-
- with pa.ipc.open_stream(b.arrow_batch.data) as reader:
- for batch in reader:
- assert isinstance(batch, pa.RecordBatch)
- batches.append(batch)
+ for attempt in Retrying(SparkConnectClient.retry_exception,
**self._retry_policy):
+ with attempt:
+ for b in self._stub.ExecutePlan(req,
metadata=self._builder.metadata()):
+ if b.client_id != self._session_id:
+ raise SparkConnectException(
+ "Received incorrect session identifier for
request."
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
