susheel-aroskar commented on code in PR #53076:
URL: https://github.com/apache/spark/pull/53076#discussion_r2540218624
##########
python/pyspark/sql/connect/client/core.py:
##########
@@ -606,6 +613,51 @@ def fromProto(cls, pb: pb2.ConfigResponse) ->
"ConfigResult":
)
+def _is_pyspark_source(filename: str) -> bool:
+ """Check if the given filename is from the pyspark package."""
+ return filename.startswith(PYSPARK_ROOT)
+
+
+def _retrieve_stack_frames() -> List[CallSite]:
+ """
+ Return a list of CallSites representing the relevant stack frames in the
callstack.
+ """
+ frames = traceback.extract_stack()
+
+ filtered_stack_frames = []
+ for i, frame in enumerate(frames):
+ filename, lineno, func, _ = frame
+ if _is_pyspark_source(filename):
+ break
+ if i + 1 < len(frames):
+ _, _, func, _ = frames[i + 1]
+ filtered_stack_frames.append(CallSite(function=func, file=filename,
linenum=lineno))
+
+ return filtered_stack_frames
+
+
+def _build_call_stack_trace() -> List[any_pb2.Any]:
+ """
+ Build a call stack trace for the current Spark Connect action
+ Returns
+ -------
+ List[any_pb2.Any]: A list of Any objects, each representing a stack frame
in the call stack trace in the user code.
+ """
+ call_stack_trace = []
+ if os.getenv("SPARK_CONNECT_DEBUG_CLIENT_CALL_STACK", "false").lower() in
("true", "1"):
Review Comment:
I feel Spark Configuration flags are mainly for things that affect Spark's -
i.e. Spark server's - behavior. This only affects client's behavior. Also if I
set it in Spark conf then I'll have to make a network call `spark.conf.get()`
every time to decide whether to include the client code call stack or not.
Anyway, that was my thinking but I'm open to changing it to Spark conf if
that's the convention / best practice.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
