holdenk commented on code in PR #53076:
URL: https://github.com/apache/spark/pull/53076#discussion_r2547124667
##########
python/pyspark/sql/connect/client/core.py:
##########
@@ -606,6 +613,51 @@ def fromProto(cls, pb: pb2.ConfigResponse) ->
"ConfigResult":
)
+def _is_pyspark_source(filename: str) -> bool:
+ """Check if the given filename is from the pyspark package."""
+ return filename.startswith(PYSPARK_ROOT)
+
+
+def _retrieve_stack_frames() -> List[CallSite]:
+ """
+ Return a list of CallSites representing the relevant stack frames in the
callstack.
+ """
+ frames = traceback.extract_stack()
+
+ filtered_stack_frames = []
+ for i, frame in enumerate(frames):
+ filename, lineno, func, _ = frame
+ if _is_pyspark_source(filename):
+ break
+ if i + 1 < len(frames):
+ _, _, func, _ = frames[i + 1]
+ filtered_stack_frames.append(CallSite(function=func, file=filename,
linenum=lineno))
+
+ return filtered_stack_frames
+
+
+def _build_call_stack_trace() -> List[any_pb2.Any]:
+ """
+ Build a call stack trace for the current Spark Connect action
+ Returns
+ -------
+ List[any_pb2.Any]: A list of Any objects, each representing a stack frame
in the call stack trace in the user code.
+ """
+ call_stack_trace = []
+ if os.getenv("SPARK_CONNECT_DEBUG_CLIENT_CALL_STACK", "false").lower() in
("true", "1"):
Review Comment:
Interesting the conf has a network call. Often in the Javaside we'd resolve
a non-modifiable config like that in a lazy val
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
