This is an automated email from the ASF dual-hosted git repository.
Yicong-Huang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3dff5f059ef2 [SPARK-60613][CORE][PYTHON][UDF] Add an init response to
the Python worker socket protocol
3dff5f059ef2 is described below
commit 3dff5f059ef21e028b20e9981009f24ed7e84b40
Author: Yicong Huang <[email protected]>
AuthorDate: Fri Oct 30 04:40:36 2026 +0000
[SPARK-60613][CORE][PYTHON][UDF] Add an init response to the Python worker
socket protocol
### What changes were proposed in this pull request?
The Python worker (`worker.py`) now sends an init response once it has
finished setting up a task (eval type, conf, UDF/UDTF/command), before any
output, and `BasePythonRunner.ReaderIterator` waits for it before reading
output.
- Frame: `START_OF_INIT_RESPONSE` (-9) followed by a length-prefixed JSON
object, mirroring the init message (`START_OF_INIT_MESSAGE`) and the JSON
metrics report. It is `{}` on success, or `{"error": {"message", "traceback",
"errorClass", "isPayloadError"}}` if initialization failed, in which case no
output follows.
- `isPayloadError` is true only while the worker deserializes the payload:
the classic command, the chained UDF functions, and the UDTF class, return type
(including the struct check) and analyze result. Task setup, including eval
type handler construction (dependency and connection checks, UDTF instantiation
and `eval` signature validation), reports false.
- Failures while processing data after the init response keep using
`PYTHON_EXCEPTION_THROWN`.
- On the JVM side, an init error surfaces as the same `PythonException`
(`PYTHON_EXCEPTION`, same message and traceback) as before, so user-facing
errors are unchanged.
### Why are the changes needed?
Part of SPARK-55278. The language-agnostic gRPC UDF protocol replies to
`Init` with exactly one `InitResponse` (success or error) before any data. The
socket protocol had no init acknowledgement, so init failures only surfaced
when the JVM read output and were indistinguishable from data-phase failures.
This aligns the two protocols and gives the JVM an explicit initialized point.
### Does this PR introduce _any_ user-facing change?
No. The JVM-to-worker protocol is internal; the JVM and `worker.py` ship
together.
### How was this patch tested?
- New `WorkerInitResponseTests` in `pyspark.tests.test_worker`: drives
`worker.main` over in-memory streams and checks the init response on success,
on payload errors of each payload kind (classic command, UDF, UDTF), on an
exception whose `str()` fails, on setup errors (Python version mismatch,
handler construction), and that a data-phase error still follows a successful
init response.
- New `BasePythonRunnerSuite` cases for decoding the init response,
including a traceback longer than Jackson's default string limit, and rejecting
malformed ones.
- Ran `pyspark.tests.test_worker`, `pyspark.sql.tests.test_udtf` and
`pyspark.sql.tests.arrow.test_arrow_python_udf`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #59860 from Yicong-Huang/pyspark-socket-init-response.
Authored-by: Yicong Huang <[email protected]>
Signed-off-by: Yicong-Huang <[email protected]>
---
.../org/apache/spark/api/python/PythonRunner.scala | 107 +++++++-
.../spark/api/python/BasePythonRunnerSuite.scala | 50 ++++
python/pyspark/serializers.py | 1 +
python/pyspark/tests/test_worker.py | 287 +++++++++++++++++++++
python/pyspark/util.py | 27 +-
python/pyspark/worker.py | 66 ++++-
6 files changed, 519 insertions(+), 19 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 0be49a5f71f0..84109d042540 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -29,8 +29,8 @@ import scala.jdk.CollectionConverters._
import scala.util.{Success, Try}
import scala.util.control.NonFatal
-import com.fasterxml.jackson.core.{JsonParser, JsonProcessingException}
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
+import com.fasterxml.jackson.core.{JsonParser, JsonProcessingException,
StreamReadConstraints}
+import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode,
ObjectMapper}
import org.apache.spark._
import org.apache.spark.api.python.PythonFunction.PythonAccumulator
@@ -477,9 +477,15 @@ private[spark] object BasePythonRunner extends Logging {
}
}
- private lazy val workerMetricsMapper = new ObjectMapper()
- .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
- .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
+ private lazy val workerJsonMapper = {
+ val mapper = new ObjectMapper()
+ .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
+ .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
+ // An init error carries the full worker traceback, which may exceed
Jackson's default limit.
+ mapper.getFactory.setStreamReadConstraints(
+ StreamReadConstraints.builder().maxStringLength(Int.MaxValue).build())
+ mapper
+ }
/** Decode every METRICS_DATA field as a named int64; reject other values. */
private[python] def readWorkerMetrics(stream: DataInputStream): Map[String,
Long] = {
@@ -489,7 +495,7 @@ private[spark] object BasePythonRunner extends Logging {
}
val json = PythonWorkerUtils.readUTF(length, stream)
val report = try {
- workerMetricsMapper.readTree(json)
+ workerJsonMapper.readTree(json)
} catch {
case e: JsonProcessingException =>
throw new SparkException("Malformed Python worker report JSON", e)
@@ -507,6 +513,45 @@ private[spark] object BasePythonRunner extends Logging {
}.toMap
}
+ /**
+ * Decode the init response that follows START_OF_INIT_RESPONSE: a
length-prefixed JSON object
+ * that is `{}` once the worker is initialized, or carries an `error` if
initialization failed.
+ * Unknown fields are ignored.
+ */
+ private[python] def readInitResponse(stream: DataInputStream):
Option[PythonWorkerInitError] = {
+ val length = stream.readInt()
+ if (length <= 0) {
+ throw new SparkException(s"Invalid Python worker init response length:
$length")
+ }
+ val json = PythonWorkerUtils.readUTF(length, stream)
+ val response = try {
+ workerJsonMapper.readTree(json)
+ } catch {
+ case e: JsonProcessingException =>
+ throw new SparkException("Malformed Python worker init response JSON",
e)
+ }
+ if (response == null || !response.isObject) {
+ throw new SparkException("Expected a Python worker init response JSON
object")
+ }
+ Option(response.get("error")).map { error =>
+ def field(name: String, isValid: JsonNode => Boolean): JsonNode = {
+ val value = error.get(name)
+ if (value == null || !isValid(value)) {
+ throw new SparkException(s"Invalid Python worker init error field:
$name")
+ }
+ value
+ }
+ if (!error.isObject) {
+ throw new SparkException("Expected a Python worker init error JSON
object")
+ }
+ PythonWorkerInitError(
+ message = field("message", _.isTextual).asText(),
+ traceback = field("traceback", _.isTextual).asText(),
+ errorClass = field("errorClass", _.isTextual).asText(),
+ isPayloadError = field("isPayloadError", _.isBoolean).asBoolean())
+ }
+ }
+
/**
* Creates a task identifier string for logging following Spark's standard
format.
* Format: "task <partition>.<attempt> in stage <stageId> (TID
<taskAttemptId>)"
@@ -1105,6 +1150,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
private var nextObj: OUT = _
private var eos = false
+ private var initialized = false
// Track batches and data size for logging
protected var batchesProcessed: Long = 0
@@ -1112,6 +1158,9 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
override def hasNext: Boolean = nextObj != null || {
if (!eos) {
+ if (!initialized) {
+ handleInitResponse()
+ }
nextObj = read()
hasNext
} else {
@@ -1136,6 +1185,30 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
*/
protected def read(): OUT
+ /**
+ * Waits for the worker's init response, which precedes all of its output,
and throws the
+ * error the worker reports if it failed to initialize. No output follows
an init error.
+ */
+ private def handleInitResponse(): Unit = {
+ if (writer.exception.isDefined) {
+ throw writer.exception.get
+ }
+ try {
+ stream.readInt() match {
+ case SpecialLengths.START_OF_INIT_RESPONSE =>
+ BasePythonRunner.readInitResponse(stream).foreach { error =>
+ logDebug(s"Python worker failed to initialize with
${error.errorClass} " +
+ s"(payload error: ${error.isPayloadError}) -
${taskIdentifier(context)}")
+ throw newPythonException(error.traceback)
+ }
+ case other =>
+ throw SparkException.internalError(
+ s"Expected the Python worker init response, but got $other")
+ }
+ } catch handleException
+ initialized = true
+ }
+
protected def handleMetricsData(): Unit = {
val workerMetrics = BasePythonRunner.readWorkerMetrics(stream)
def requiredMetric(name: String): Long = workerMetrics.getOrElse(name,
@@ -1188,7 +1261,10 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
protected def handlePythonException(): PythonException = {
// Signals that an exception has been thrown in python
- val traceback = PythonWorkerUtils.readUTF(stream)
+ newPythonException(PythonWorkerUtils.readUTF(stream))
+ }
+
+ private def newPythonException(traceback: String): PythonException = {
val msg = "An exception was thrown from the Python worker"
new PythonException(
errorClass = "PYTHON_EXCEPTION",
@@ -1761,8 +1837,25 @@ private[spark] object SpecialLengths {
val START_ARROW_STREAM = -6
val END_OF_MICRO_BATCH = -7
val START_OF_INIT_MESSAGE = -8
+ val START_OF_INIT_RESPONSE = -9
}
+/**
+ * An initialization failure reported by the Python worker in its init
response.
+ *
+ * @param message the exception message
+ * @param traceback the formatted exception, as reported for a failure while
processing data
+ * @param errorClass the Python exception class name, e.g. "ValueError"
+ * @param isPayloadError whether the worker could not deserialize the UDF
payload (e.g. it does
+ * not unpickle); setting up the task, including
constructing the eval type
+ * handler and its dependency and connection checks, is
not payload loading
+ */
+private[spark] case class PythonWorkerInitError(
+ message: String,
+ traceback: String,
+ errorClass: String,
+ isPayloadError: Boolean)
+
private[spark] object BarrierTaskContextMessageProtocol {
val BARRIER_FUNCTION = 1
val ALL_GATHER_FUNCTION = 2
diff --git
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
index f4a9d11e967c..cdb9afef968b 100644
---
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.{AtomicInteger,
AtomicReference}
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.duration._
+import com.fasterxml.jackson.core.StreamReadConstraints
import com.google.common.util.concurrent.Uninterruptibles
import org.apache.logging.log4j.Level
import org.scalatest.concurrent.Eventually.{eventually, timeout}
@@ -833,4 +834,53 @@ class BasePythonRunnerSuite extends SparkFunSuite with
SharedSparkContext {
assert(result == expectedMetrics + ("futureMetric" -> 17L))
checkEndMarkers(stream)
}
+
+ private val initErrorJson =
+ "{\"error\":{\"message\":\"bad\",\"traceback\":\"Traceback
...\\nValueError: bad\"," +
+ "\"errorClass\":\"ValueError\",\"isPayloadError\":true}}"
+
+ test("SPARK-60613: read an init response and leave the following sections
aligned") {
+ Seq("{}", "{\"futureField\":1}").foreach { json =>
+ val stream = framedMetricsStream(json)
+ assert(BasePythonRunner.readInitResponse(stream).isEmpty)
+ checkEndMarkers(stream)
+ }
+ }
+
+ test("SPARK-60613: decode the init error in an init response") {
+ val stream = framedMetricsStream(initErrorJson)
+ assert(BasePythonRunner.readInitResponse(stream) ===
Some(PythonWorkerInitError(
+ message = "bad",
+ traceback = "Traceback ...\nValueError: bad",
+ errorClass = "ValueError",
+ isPayloadError = true)))
+ checkEndMarkers(stream)
+ }
+
+ test("SPARK-60613: decode an init error longer than Jackson's default string
limit") {
+ val traceback = "x" * (StreamReadConstraints.DEFAULT_MAX_STRING_LEN + 1)
+ val json = initErrorJson.replace("Traceback ...\\nValueError: bad",
traceback)
+ val stream = framedMetricsStream(json)
+ assert(BasePythonRunner.readInitResponse(stream).map(_.traceback) ===
Some(traceback))
+ checkEndMarkers(stream)
+ }
+
+ test("SPARK-60613: init response rejects malformed JSON and invalid init
errors") {
+ val invalid = Seq("[]", "null", "{", "{}{}", "{\"error\":1}",
"{\"error\":null}",
+ initErrorJson.replace("\"message\":\"bad\",", ""),
+ initErrorJson.replace("\"errorClass\":\"ValueError\"",
"\"errorClass\":1"),
+ initErrorJson.replace("\"isPayloadError\":true",
"\"isPayloadError\":\"true\""))
+ invalid.foreach { json =>
+ val stream = framedMetricsStream(json)
+ intercept[SparkException] {
+ BasePythonRunner.readInitResponse(stream)
+ }
+ checkEndMarkers(stream)
+ }
+ Seq(-1, 0).foreach { length =>
+ intercept[SparkException] {
+ BasePythonRunner.readInitResponse(framedMetricsStream("{}",
Some(length)))
+ }
+ }
+ }
}
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index ca6aecbd706d..061e066f2830 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -93,6 +93,7 @@ class SpecialLengths:
NULL = -5
START_ARROW_STREAM = -6
START_OF_INIT_MESSAGE = -8
+ START_OF_INIT_RESPONSE = -9
class Serializer:
diff --git a/python/pyspark/tests/test_worker.py
b/python/pyspark/tests/test_worker.py
index 48d2fc48e9f9..5d1ace34c44f 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import io
+import json
import os
import signal
import sys
@@ -21,6 +23,7 @@ import tempfile
import threading
import time
import unittest
+from typing import Any, Optional
from unittest.mock import patch
has_resource_module = True
@@ -32,9 +35,26 @@ except ImportError:
from py4j.protocol import Py4JJavaError
from pyspark import SparkConf, SparkContext
+from pyspark.accumulators import _accumulatorRegistry
+from pyspark.core.files import SparkFiles
+from pyspark.serializers import (
+ CloudPickleSerializer,
+ CPickleSerializer,
+ SpecialLengths,
+ read_int,
+ write_int,
+ write_long,
+ write_with_length,
+)
+from pyspark.sql.types import LongType, StructField, StructType
+from pyspark.taskcontext import TaskContext
from pyspark.testing.utils import QuietTest, ReusedPySparkTestCase, eventually
+from pyspark.util import PythonEvalType
from pyspark.worker_metrics import WorkerMetrics
+with patch.dict(os.environ, {"SPARK_PYTHON_RUNTIME": "PYTHON_WORKER"}):
+ from pyspark.worker import main as worker_main
+
class WorkerMetricsTests(unittest.TestCase):
def setUp(self):
@@ -126,6 +146,273 @@ class WorkerMetricsTests(unittest.TestCase):
self.assertEqual(metrics.to_dict(), {"duration": 3})
+class WorkerInitResponseTests(unittest.TestCase):
+ """Tests for the init response the worker sends before any output, run
without a JVM."""
+
+ def setUp(self):
+ # The worker runs in this process and sets up process-wide state as a
worker would.
+ saved_files = (SparkFiles._root_directory,
SparkFiles._is_running_on_worker)
+ saved_path = list(sys.path)
+ saved_accumulators = dict(_accumulatorRegistry)
+
+ def restore():
+ TaskContext._setTaskContext(None)
+ SparkFiles._root_directory, SparkFiles._is_running_on_worker =
saved_files
+ sys.path[:] = saved_path
+ _accumulatorRegistry.clear()
+ _accumulatorRegistry.update(saved_accumulators)
+
+ self.addCleanup(restore)
+
+ @staticmethod
+ def _framed(payload: bytes) -> bytes:
+ buf = io.BytesIO()
+ write_with_length(payload, buf)
+ return buf.getvalue()
+
+ @staticmethod
+ def _worker_input(
+ udf_info: bytes,
+ items: list,
+ python_version: Optional[str] = None,
+ eval_type: int = PythonEvalType.NON_UDF,
+ ) -> bytes:
+ """Build the socket input of a task of ``eval_type`` whose init
message ends with
+ ``udf_info``, followed by ``items`` as pickled input."""
+ init = io.BytesIO()
+ write_int(0, init) # split_index
+ write_with_length((python_version or "%d.%d" %
sys.version_info[:2]).encode(), init)
+ task_context = {
+ "isBarrier": False,
+ "connInfo": None,
+ "secret": None,
+ "stageId": 0,
+ "partitionId": 0,
+ "attemptNumber": 0,
+ "taskAttemptId": 0,
+ "cpus": 1,
+ "cpuAmount": "1",
+ "resources": {},
+ "localProperties": {},
+ }
+ write_with_length(json.dumps(task_context).encode(), init)
+ write_with_length(tempfile.gettempdir().encode(), init) #
spark_files_dir
+ write_int(0, init) # num_python_includes
+ init.write(b"\x00") # needs_broadcast_decryption_server
+ write_int(0, init) # num_broadcast_variables
+ write_int(eval_type, init)
+ write_int(0, init) # runner conf
+ write_int(0, init) # eval conf
+ init.write(udf_info)
+
+ buf = io.BytesIO()
+ write_int(SpecialLengths.START_OF_INIT_MESSAGE, buf)
+ write_with_length(init.getvalue(), buf)
+ CPickleSerializer().dump_stream(iter(items), buf)
+ write_int(SpecialLengths.END_OF_DATA_SECTION, buf)
+ write_int(SpecialLengths.END_OF_STREAM, buf)
+ return buf.getvalue()
+
+ @classmethod
+ def _command(cls, func: Any) -> bytes:
+ """The udf_info of a classic RDD (NON_UDF) task running ``func``."""
+ ser = CPickleSerializer()
+ return cls._framed(CloudPickleSerializer().dumps((func, None, ser,
ser)))
+
+ @staticmethod
+ def _udf_info(func: Any, command: Optional[bytes] = None) -> bytes:
+ """The udf_info of a task running one single-argument UDF ``func``, or
the pickled
+ ``command`` in its place."""
+ if command is None:
+ command = CloudPickleSerializer().dumps((func, LongType()))
+ buf = io.BytesIO()
+ write_int(1, buf) # num_udfs
+ write_int(1, buf) # num_args
+ write_int(0, buf) # arg offset
+ buf.write(b"\x00") # is_kwarg
+ write_int(1, buf) # num_chained
+ write_with_length(command, buf)
+ write_long(0, buf) # result_id
+ return buf.getvalue()
+
+ @staticmethod
+ def _udtf_info(udtf_cls: type, return_type: str) -> bytes:
+ """The udf_info of a task running ``udtf_cls`` on one argument with
``return_type``."""
+ buf = io.BytesIO()
+ write_int(1, buf) # num_args
+ write_int(0, buf) # arg offset
+ buf.write(b"\x00") # is_kwarg
+ write_int(0, buf) # num_partition_child_indexes
+ buf.write(b"\x00") # has_pickled_analyze_result
+ write_with_length(CloudPickleSerializer().dumps(udtf_cls), buf)
+ write_with_length(return_type.encode(), buf)
+ write_with_length(b"test_udtf", buf) # name
+ return buf.getvalue()
+
+ @classmethod
+ def _raising_payload(cls, error: BaseException) -> bytes:
+ """A NON_UDF payload whose unpickling raises ``error`` from
``fail_on_unpickle``."""
+
+ def fail_on_unpickle():
+ raise error
+
+ class RaisesOnUnpickle:
+ def __reduce__(self):
+ return fail_on_unpickle, ()
+
+ return cls._framed(CloudPickleSerializer().dumps(RaisesOnUnpickle()))
+
+ def _run(self, worker_input: bytes, expect_exit: bool = False) ->
io.BytesIO:
+ outfile = io.BytesIO()
+ if expect_exit:
+ with self.assertRaises(SystemExit):
+ worker_main(io.BytesIO(worker_input), outfile)
+ else:
+ worker_main(io.BytesIO(worker_input), outfile)
+ outfile.seek(0)
+ return outfile
+
+ def _read_init_response(self, outfile: io.BytesIO) -> dict:
+ self.assertEqual(read_int(outfile),
SpecialLengths.START_OF_INIT_RESPONSE)
+ return json.loads(outfile.read(read_int(outfile)).decode("utf-8"))
+
+ def test_init_response_precedes_output(self):
+ worker_input = self._worker_input(self._command(lambda _, it: (x * 2
for x in it)), [1, 2])
+ outfile = self._run(worker_input)
+ self.assertEqual(self._read_init_response(outfile), {})
+ ser = CPickleSerializer()
+ self.assertEqual([ser._read_with_length(outfile) for _ in range(2)],
[2, 4])
+ self.assertEqual(read_int(outfile), SpecialLengths.METRICS_DATA)
+
+ def test_payload_error_is_reported_in_init_response(self):
+ worker_input = self._worker_input(self._framed(b"not a pickle"), [1])
+ outfile = self._run(worker_input, expect_exit=True)
+ error = self._read_init_response(outfile)["error"]
+ self.assertTrue(error["isPayloadError"])
+ self.assertEqual(error["errorClass"], "UnpicklingError")
+ self.assertIn("UnpicklingError", error["traceback"])
+ self.assertTrue(error["message"])
+ # No output follows an init error.
+ self.assertEqual(outfile.read(), b"")
+
+ def test_payload_error_with_unprintable_exception(self):
+ class UnprintableError(Exception):
+ def __str__(self):
+ raise RuntimeError("str() failed")
+
+ worker_input =
self._worker_input(self._raising_payload(UnprintableError()), [1])
+ outfile = self._run(worker_input, expect_exit=True)
+ error = self._read_init_response(outfile)["error"]
+ self.assertEqual(
+ error,
+ {
+ "message": "<exception str() failed>",
+ "traceback": error["traceback"],
+ "errorClass": "UnprintableError",
+ "isPayloadError": True,
+ },
+ )
+ self.assertIn("UnprintableError", error["traceback"])
+ self.assertEqual(outfile.read(), b"")
+
+ def test_init_error_traceback_follows_worker_settings(self):
+ worker_input =
self._worker_input(self._raising_payload(ValueError("bad payload")), [1])
+ no_traceback_settings = {
+ "SPARK_HIDE_TRACEBACK": "",
+ "SPARK_SIMPLIFIED_TRACEBACK": "",
+ "SPARK_TRACEBACK_WITH_LOCALS": "",
+ }
+ # By default the traceback keeps the frames of the handled exception.
+ with patch.dict(os.environ, no_traceback_settings):
+ outfile = self._run(worker_input, expect_exit=True)
+ tb = self._read_init_response(outfile)["error"]["traceback"]
+ self.assertTrue(tb.startswith("Traceback (most recent call last):"))
+ self.assertIn("in fail_on_unpickle", tb)
+ self.assertTrue(tb.endswith("ValueError: bad payload\n"))
+
+ with patch.dict(os.environ, {**no_traceback_settings,
"SPARK_HIDE_TRACEBACK": "1"}):
+ outfile = self._run(worker_input, expect_exit=True)
+ error = self._read_init_response(outfile)["error"]
+ self.assertEqual(error["traceback"], "ValueError: bad payload\n")
+
+ def test_setup_error_is_reported_in_init_response(self):
+ worker_input = self._worker_input(
+ self._command(lambda _, it: it), [1], python_version="2.0"
+ )
+ outfile = self._run(worker_input, expect_exit=True)
+ error = self._read_init_response(outfile)["error"]
+ self.assertFalse(error["isPayloadError"])
+ self.assertEqual(error["errorClass"], "PySparkRuntimeError")
+ self.assertIn("PYTHON_VERSION_MISMATCH", error["traceback"])
+ self.assertEqual(outfile.read(), b"")
+
+ def test_handler_construction_error_is_not_a_payload_error(self):
+ class FailingHandler:
+ def __init__(self, **kwargs):
+ raise ImportError("pyarrow is not installed")
+
+ worker_input = self._worker_input(
+ self._udf_info(lambda x: x), [1],
eval_type=PythonEvalType.SQL_BATCHED_UDF
+ )
+ with patch("pyspark.worker.get_eval_type_handler",
return_value=FailingHandler):
+ outfile = self._run(worker_input, expect_exit=True)
+ error = self._read_init_response(outfile)["error"]
+ self.assertFalse(error["isPayloadError"])
+ self.assertEqual(error["errorClass"], "ImportError")
+ self.assertEqual(error["message"], "pyarrow is not installed")
+ self.assertEqual(outfile.read(), b"")
+
+ def test_payload_window_of_each_payload_kind(self):
+ class TwoArgUDTF:
+ def eval(self, a, b):
+ yield (a,)
+
+ struct_type = StructType([StructField("x", LongType())]).json()
+ cases = [
+ # A UDF whose function does not unpickle.
+ (
+ PythonEvalType.SQL_BATCHED_UDF,
+ self._udf_info(None, command=b"not a pickle"),
+ "UnpicklingError",
+ True,
+ ),
+ # A UDTF that declares a non-struct return type.
+ (
+ PythonEvalType.SQL_TABLE_UDF,
+ self._udtf_info(TwoArgUDTF, '"long"'),
+ "must be a struct type",
+ True,
+ ),
+ # A UDTF whose 'eval' signature does not match the call, checked
by its handler.
+ (
+ PythonEvalType.SQL_TABLE_UDF,
+ self._udtf_info(TwoArgUDTF, struct_type),
+ "UDTF_EVAL_METHOD_ARGUMENTS_DO_NOT_MATCH_SIGNATURE",
+ False,
+ ),
+ ]
+ for eval_type, udf_info, expected_error, is_payload_error in cases:
+ with self.subTest(expected_error=expected_error):
+ outfile = self._run(
+ self._worker_input(udf_info, [1], eval_type=eval_type),
expect_exit=True
+ )
+ error = self._read_init_response(outfile)["error"]
+ self.assertIn(expected_error, error["traceback"])
+ self.assertEqual(error["isPayloadError"], is_payload_error)
+ self.assertEqual(outfile.read(), b"")
+
+ def test_data_error_is_reported_after_init_response(self):
+ def fail(_, it):
+ for _x in it:
+ raise ValueError("data error")
+ yield
+
+ outfile = self._run(self._worker_input(self._command(fail), [1]),
expect_exit=True)
+ self.assertEqual(self._read_init_response(outfile), {})
+ self.assertEqual(read_int(outfile),
SpecialLengths.PYTHON_EXCEPTION_THROWN)
+ self.assertIn("data error",
outfile.read(read_int(outfile)).decode("utf-8"))
+
+
class WorkerTests(ReusedPySparkTestCase):
def test_cancel_task(self):
temp = tempfile.NamedTemporaryFile(delete=True)
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index cdfdb432f006..0b7f910a9fbe 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -529,6 +529,22 @@ def _format_exception(
return "".join(te.format())
+def format_worker_exception(e: BaseException, hide_traceback: Optional[bool] =
None) -> str:
+ """
+ Formats an exception raised in the Python worker as it is reported to the
JVM, honoring the
+ SPARK_HIDE_TRACEBACK, SPARK_SIMPLIFIED_TRACEBACK and
SPARK_TRACEBACK_WITH_LOCALS environment
+ variables. Must be called while handling ``e``.
+ """
+ if hide_traceback is None:
+ hide_traceback = bool(os.environ.get("SPARK_HIDE_TRACEBACK", False))
+ return _format_exception(
+ e,
+ hide_traceback,
+ bool(os.environ.get("SPARK_SIMPLIFIED_TRACEBACK", False)),
+ bool(os.environ.get("SPARK_TRACEBACK_WITH_LOCALS", False)),
+ )
+
+
def handle_worker_exception(
e: BaseException, outfile: IO, hide_traceback: Optional[bool] = None
) -> None:
@@ -547,17 +563,8 @@ def handle_worker_exception(
Whether to hide the traceback in the output.
By default, hides the traceback if environment variable
SPARK_HIDE_TRACEBACK is set.
"""
-
- if hide_traceback is None:
- hide_traceback = bool(os.environ.get("SPARK_HIDE_TRACEBACK", False))
-
try:
- exc_info = _format_exception(
- e,
- hide_traceback,
- bool(os.environ.get("SPARK_SIMPLIFIED_TRACEBACK", False)),
- bool(os.environ.get("SPARK_TRACEBACK_WITH_LOCALS", False)),
- )
+ exc_info = format_worker_exception(e, hide_traceback)
write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, outfile)
write_with_length(exc_info.encode("utf-8"), outfile)
except IOError:
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index dc20ee355c7d..f10e0c0dbfad 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -26,8 +26,9 @@ through the handler registered for their eval type in
pyspark.eval_handlers.
import json
import os
import sys
+import traceback
import warnings
-from typing import BinaryIO
+from typing import Any, BinaryIO, Optional
from pyspark import shuffle
from pyspark.accumulators import _accumulatorRegistry
@@ -49,6 +50,7 @@ from pyspark.sql.types import StructType,
_parse_datatype_json_string
from pyspark.taskcontext import BarrierTaskContext, TaskContext
from pyspark.util import (
PythonEvalType,
+ format_worker_exception,
handle_worker_exception,
start_faulthandler_periodic_traceback,
with_faulthandler,
@@ -76,6 +78,41 @@ def report_metrics(outfile: BinaryIO, metrics: dict[str,
int]) -> None:
write_with_length(payload, outfile)
+def _exception_message(e: BaseException) -> str:
+ """``str(e)``, or the placeholder ``traceback`` uses when the conversion
fails."""
+ try:
+ return str(e)
+ except BaseException:
+ return "<exception str() failed>"
+
+
+def report_init_response(
+ outfile: BinaryIO, error: Optional[BaseException] = None,
is_payload_error: bool = False
+) -> None:
+ """
+ Write the init response as a length-prefixed JSON object after
START_OF_INIT_RESPONSE.
+
+ It is sent exactly once per task, before any output: ``{}`` once the task
is initialized, or
+ ``{"error": {...}}`` if initialization failed, in which case no output
follows. An init error
+ carries the exception message, the traceback as the JVM reports it, the
exception class name,
+ and whether the worker could not deserialize the UDF payload
(``isPayloadError``), e.g. the
+ command does not unpickle or a UDTF declares a non-struct return type.
Setting up the task,
+ including constructing the eval type handler and its dependency and
connection checks, is not
+ payload loading. Must be called while handling ``error``.
+ """
+ response: dict[str, Any] = {}
+ if error is not None:
+ response["error"] = {
+ "message": _exception_message(error),
+ "traceback": format_worker_exception(error),
+ "errorClass": type(error).__name__,
+ "isPayloadError": is_payload_error,
+ }
+ payload = json.dumps(response, separators=(",", ":")).encode("utf-8")
+ write_int(SpecialLengths.START_OF_INIT_RESPONSE, outfile)
+ write_with_length(payload, outfile)
+
+
def invoke_udf(message_receiver: SparkMessageReceiver, outfile: BinaryIO):
"""
This function is the main processing function for worker.py.
@@ -84,7 +121,13 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
This method goes through three phases:
Initialization -> Processing -> Finish/Cleanup
+
+ Initialization ends with the init response (see ``report_init_response``);
a failure before
+ it is reported there, and a failure while processing data after it as
+ PYTHON_EXCEPTION_THROWN.
"""
+ initialized = False
+ loading_payload = False
try:
metrics = WorkerMetrics()
metrics.reset()
@@ -122,7 +165,9 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
profiler = None
if eval_type == PythonEvalType.NON_UDF:
assert isinstance(init_info.udf_info, (bytes, memoryview))
+ loading_payload = True
func, profiler, deserializer, serializer = read_command(pickleSer,
init_info.udf_info)
+ loading_payload = False
else:
# UDF and UDTF runners dispatch through the eval type's registered
handler, which
# provides both the function and the single serializer for input
and output.
@@ -132,6 +177,7 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
handler: EvalTypeHandler
if isinstance(init_info.udf_info, UDTFInfo):
assert issubclass(handler_cls, UDTFHandlerBase)
+ loading_payload = True
udtf_cls = read_command(pickleSer, init_info.udf_info.handler)
return_type =
_parse_datatype_json_string(init_info.udf_info.return_type)
if not isinstance(return_type, StructType):
@@ -152,11 +198,13 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
analyze_result=analyze_result,
name=init_info.udf_info.name,
)
+ loading_payload = False
handler = handler_cls(
udf_descs=[udtf], runner_conf=runner_conf,
eval_conf=eval_conf
)
else:
assert isinstance(init_info.udf_info, list)
+ loading_payload = True
udf_descs = []
for udf_info in init_info.udf_info:
# The functions applied in turn to one column; the last
one's return type
@@ -171,6 +219,7 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
result_id=udf_info.result_id,
)
)
+ loading_payload = False
handler = handler_cls(
udf_descs=udf_descs, runner_conf=runner_conf,
eval_conf=eval_conf
)
@@ -181,6 +230,8 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
del init_info
metrics.set_current_timestamp("initTimestampMs")
+ report_init_response(outfile)
+ initialized = True
# Processing
@@ -305,7 +356,18 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
TaskContext._setTaskContext(None)
BarrierTaskContext._setTaskContext(None)
except BaseException as e:
- handle_worker_exception(e, outfile)
+ if initialized:
+ handle_worker_exception(e, outfile)
+ else:
+ try:
+ report_init_response(outfile, e, loading_payload)
+ except IOError:
+ # JVM close the socket
+ pass
+ except BaseException:
+ # Write the error to stderr if it happened while serializing
+ print("PySpark worker failed with exception:", file=sys.stderr)
+ print(traceback.format_exc(), file=sys.stderr)
sys.exit(-1)
metrics.set_current_timestamp("finishTimestampMs")
# Spill totals are byte counts already expressed in their reporting unit.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]