This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a646dc80468c [SPARK-58119][CORE][PYTHON] Introduce PythonWorkerHandle
abstraction for the Python worker path
a646dc80468c is described below
commit a646dc80468ce2b9e3e1ca3ff3191b30d071a14f
Author: Fabian Paul <[email protected]>
AuthorDate: Mon Jul 27 11:43:28 2026 -0700
[SPARK-58119][CORE][PYTHON] Introduce PythonWorkerHandle abstraction for
the Python worker path
### What changes were proposed in this pull request?
This PR introduces a small `PythonWorkerHandle` trait in
`org.apache.spark.api.python` that
abstracts the concrete Python-worker process backend, exposing only what
the runner needs:
- `isAlive(): Boolean`
- `destroy(): Boolean`
- `terminationDiagnostics(): Option[String]`
`LocalPythonWorkerHandle` is the implementation backed by a local OS process
(`java.lang.ProcessHandle`). Its
`terminationDiagnostics` reads the worker's Python faulthandler log at
`<faultHandlerLogDir>/<pid>`
(read-once, deleting the file after reading), where `faultHandlerLogDir:
Option[File]` is bound when
the handle is created (`None` means faulthandler output is disabled for
that worker).
`PythonWorkerHandle.of(pid, faultHandlerLogDir)` constructs one.
`PythonWorkerFactory` derives `faultHandlerLogDir` from `envVars` (the
`PYTHON_FAULTHANDLER_DIR`
entry it already passes to the worker process) and builds the handle. As a
result the reader path no
longer needs a separate `faultHandlerEnabled` flag or a pre-projected pid
threaded through it:
`SparkEnv.createPythonWorker`, `BasePythonRunner` (`ReaderIterator`,
`ReaderInputStream`,
`createPipelinedDataIn`, `newReaderIterator`,
`pythonWorkerStatusMessageWithContext`),
`PythonUDFRunner`, `PythonArrowOutput`, and `PythonPlannerRunner` now carry
`Option[PythonWorkerHandle]`, and the crash / idle-timeout messages call
`handle.flatMap(_.terminationDiagnostics())`. The now-unused
`BasePythonRunner.tryReadFaultHandlerLog` / `faultHandlerLogPath` helpers
are removed (their logic
now lives in `LocalPythonWorkerHandle`).
### Why are the changes needed?
Currently the runner depends directly on `java.lang.ProcessHandle` and
spreads two separate concerns
across the reader path: how to talk to the worker process (liveness /
termination) and how to obtain
its post-mortem diagnostics (the faulthandler log, gated by a separate
boolean and a pre-projected
pid). Abstracting the worker behind a handle decouples the runner from the
concrete OS-process type,
centralizes diagnostics ownership on the handle, and makes the worker
backend unit-testable.
### Does this PR introduce _any_ user-facing change?
No. This is an internal refactor; worker lifecycle and the existing crash /
idle-timeout error
messages are unchanged.
### How was this patch tested?
Added `PythonWorkerHandleSuite`, covering `isAlive`/`destroy`,
`PythonWorkerHandle.of`, and
`terminationDiagnostics` (log present, log absent, and disabled when no log
directory is bound).
Built and test-compiled `core` and `sql/core` locally; the existing Python
worker test suites
exercise the reader path end to end.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: generative AI tooling (assisted authoring and refactoring).
Closes #57247 from fapaul/processhandle-oss-demo.
Authored-by: Fabian Paul <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
---
.../src/main/scala/org/apache/spark/SparkEnv.scala | 8 +-
.../org/apache/spark/api/python/PythonRunner.scala | 50 ++++--------
.../spark/api/python/PythonWorkerFactory.scala | 28 ++++---
.../spark/api/python/PythonWorkerHandle.scala | 94 ++++++++++++++++++++++
.../spark/api/python/PythonWorkerHandleSuite.scala | 84 +++++++++++++++++++
.../sql/execution/python/PythonArrowOutput.scala | 6 +-
.../sql/execution/python/PythonPlannerRunner.scala | 16 ++--
.../sql/execution/python/PythonUDFRunner.scala | 4 +-
8 files changed, 227 insertions(+), 63 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala
b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index 4cc2c45d7ee8..ca48ee473eb0 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -30,7 +30,7 @@ import com.google.common.cache.CacheBuilder
import org.apache.hadoop.conf.Configuration
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.api.python.{PythonWorker, PythonWorkerFactory}
+import org.apache.spark.api.python.{PythonWorker, PythonWorkerFactory,
PythonWorkerHandle}
import org.apache.spark.broadcast.BroadcastManager
import org.apache.spark.executor.ExecutorBackend
import org.apache.spark.internal.{config, Logging}
@@ -312,7 +312,7 @@ class SparkEnv (
workerModule: String,
daemonModule: String,
envVars: Map[String, String],
- useDaemon: Boolean): (PythonWorker, Option[ProcessHandle]) = {
+ useDaemon: Boolean): (PythonWorker, Option[PythonWorkerHandle]) = {
synchronized {
val key = PythonWorkersKey(pythonExec, workerModule, daemonModule,
envVars)
val workerFactory = pythonWorkers.getOrElseUpdate(key, new
PythonWorkerFactory(
@@ -331,7 +331,7 @@ class SparkEnv (
pythonExec: String,
workerModule: String,
envVars: Map[String, String],
- useDaemon: Boolean): (PythonWorker, Option[ProcessHandle]) = {
+ useDaemon: Boolean): (PythonWorker, Option[PythonWorkerHandle]) = {
createPythonWorker(
pythonExec, workerModule, PythonWorkerFactory.defaultDaemonModule,
envVars, useDaemon)
}
@@ -340,7 +340,7 @@ class SparkEnv (
pythonExec: String,
workerModule: String,
daemonModule: String,
- envVars: Map[String, String]): (PythonWorker, Option[ProcessHandle]) = {
+ envVars: Map[String, String]): (PythonWorker,
Option[PythonWorkerHandle]) = {
val useDaemon = conf.get(Python.PYTHON_USE_DAEMON)
createPythonWorker(
pythonExec, workerModule, daemonModule, envVars, useDaemon)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 3c300be6a7fc..20424e4f37df 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -21,7 +21,6 @@ import java.io._
import java.net._
import java.nio.ByteBuffer
import java.nio.channels.{AsynchronousCloseException, Channels, SelectionKey,
ServerSocketChannel, SocketChannel}
-import java.nio.file.{Files => JavaFiles, Path}
import java.util.UUID
import java.util.concurrent.{CancellationException, ConcurrentHashMap,
ExecutionException, TimeUnit}
import java.util.concurrent.atomic.AtomicBoolean
@@ -139,22 +138,6 @@ private[spark] object BasePythonRunner extends Logging {
private[spark] lazy val faultHandlerLogDir = Utils.createTempDir(namePrefix
= "faulthandler")
- private[spark] def faultHandlerLogPath(pid: Int): Path = {
- new File(faultHandlerLogDir, pid.toString).toPath
- }
-
- private[spark] def tryReadFaultHandlerLog(
- faultHandlerEnabled: Boolean, pid: Option[Int]): Option[String] = {
- if (faultHandlerEnabled) {
- pid.map(faultHandlerLogPath).collect {
- case path if JavaFiles.exists(path) =>
- val error = String.join("\n", JavaFiles.readAllLines(path)) + "\n"
- JavaFiles.deleteIfExists(path)
- error
- }
- } else None
- }
-
/**
* Splits the executor-wide pyspark memory allocation evenly across the
executor's task slots.
* The Python worker pool can grow to the number of tasks that run
concurrently on the executor
@@ -193,11 +176,11 @@ private[spark] object BasePythonRunner extends Logging {
}
private[spark] def pythonWorkerStatusMessageWithContext(
- handle: Option[ProcessHandle],
+ handle: Option[PythonWorkerHandle],
worker: PythonWorker,
hasInputs: Boolean): MessageWithContext = {
log"handle.map(_.isAlive) = " +
- log"${MDC(LogKeys.PYTHON_WORKER_IS_ALIVE, handle.map(_.isAlive))}, " +
+ log"${MDC(LogKeys.PYTHON_WORKER_IS_ALIVE, handle.map(_.isAlive()))}, " +
log"channel.isConnected = " +
log"${MDC(LogKeys.PYTHON_WORKER_CHANNEL_IS_CONNECTED,
worker.channel.isConnected)}, " +
log"channel.isBlocking = " +
@@ -390,7 +373,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
envVars.put("SPARK_PIPELINED_UDF_QUEUE_DEPTH",
pipelinedQueueDepth.toString)
}
- val (worker: PythonWorker, handle: Option[ProcessHandle]) =
env.createPythonWorker(
+ val (worker: PythonWorker, handle: Option[PythonWorkerHandle]) =
env.createPythonWorker(
pythonExec, workerModule, daemonModule, envVars.asScala.toMap, useDaemon)
// Whether is the worker released into idle pool or closed. When any codes
try to release or
// close a worker, they should use `releasedOrClosed.compareAndSet` to
flip the state to make
@@ -428,11 +411,11 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
} else {
new DataInputStream(new BufferedInputStream(
new ReaderInputStream(worker, writer, handle,
- faultHandlerEnabled, idleTimeoutSeconds, killOnIdleTimeout, context),
+ idleTimeoutSeconds, killOnIdleTimeout, context),
bufferSize))
}
val stdoutIterator = newReaderIterator(
- dataIn, writer, startTime, env, worker, handle.map(_.pid.toInt),
releasedOrClosed, context)
+ dataIn, writer, startTime, env, worker, handle, releasedOrClosed,
context)
new InterruptibleIterator(context, stdoutIterator)
}
@@ -443,7 +426,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
private def createPipelinedDataIn(
worker: PythonWorker,
writer: Writer,
- handle: Option[ProcessHandle],
+ handle: Option[PythonWorkerHandle],
context: TaskContext): DataInputStream = {
// Switch the channel to blocking mode for true full-duplex I/O.
// The channel is left in blocking mode after the task completes; with
worker reuse
@@ -532,7 +515,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
log" - ${MDC(TASK_NAME, taskIdentifier(context))}")
if (killOnIdleTimeout) {
handle.foreach { h =>
- if (h.isAlive) {
+ if (h.isAlive()) {
logWarning(
log"Terminating Python worker process due to idle
timeout " +
log"(timeout: " +
@@ -548,7 +531,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
if (result == -1 && pythonWorkerKilled) {
val base = "Python worker process terminated due to idle timeout " +
s"(timeout: $idleTimeoutSeconds seconds)"
- val msg = tryReadFaultHandlerLog(faultHandlerEnabled,
handle.map(_.pid.toInt))
+ val msg = handle.flatMap(_.terminationDiagnostics())
.map(error => s"$base: $error")
.getOrElse(base)
throw new PythonWorkerException(msg)
@@ -572,7 +555,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
startTime: Long,
env: SparkEnv,
worker: PythonWorker,
- pid: Option[Int],
+ handle: Option[PythonWorkerHandle],
releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[OUT]
@@ -792,7 +775,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
startTime: Long,
env: SparkEnv,
worker: PythonWorker,
- pid: Option[Int],
+ handle: Option[PythonWorkerHandle],
releasedOrClosed: AtomicBoolean,
context: TaskContext)
extends Iterator[OUT] {
@@ -912,7 +895,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
case e: IOException =>
val base = "Python worker exited unexpectedly (crashed)"
- val msg = tryReadFaultHandlerLog(faultHandlerEnabled, pid)
+ val msg = handle.flatMap(_.terminationDiagnostics())
.map(error => s"$base: $error")
.getOrElse(base)
throw new SparkException(msg, e)
@@ -974,8 +957,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
class ReaderInputStream(
worker: PythonWorker,
writer: Writer,
- handle: Option[ProcessHandle],
- faultHandlerEnabled: Boolean,
+ handle: Option[PythonWorkerHandle],
idleTimeoutSeconds: Long,
killOnIdleTimeout: Boolean,
context: TaskContext) extends InputStream {
@@ -1061,7 +1043,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
log" - ${MDC(TASK_NAME, taskIdentifier(context))}")
if (killOnIdleTimeout) {
handle.foreach { handle =>
- if (handle.isAlive) {
+ if (handle.isAlive()) {
logWarning(log"Terminating Python worker process due to idle
timeout " +
log"(timeout: ${MDC(PYTHON_WORKER_IDLE_TIMEOUT,
idleTimeoutSeconds)} " +
log"seconds) - ${MDC(TASK_NAME, taskIdentifier(context))}")
@@ -1118,7 +1100,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
if (n == -1 && pythonWorkerKilled) {
val base = "Python worker process terminated due to idle timeout " +
s"(timeout: $idleTimeoutSeconds seconds)"
- val msg = tryReadFaultHandlerLog(faultHandlerEnabled,
handle.map(_.pid.toInt))
+ val msg = handle.flatMap(_.terminationDiagnostics())
.map(error => s"$base: $error")
.getOrElse(base)
throw new PythonWorkerException(msg)
@@ -1352,11 +1334,11 @@ private[spark] class PythonRunner(
startTime: Long,
env: SparkEnv,
worker: PythonWorker,
- pid: Option[Int],
+ handle: Option[PythonWorkerHandle],
releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[Array[Byte]] = {
new ReaderIterator(
- stream, writer, startTime, env, worker, pid, releasedOrClosed, context) {
+ stream, writer, startTime, env, worker, handle, releasedOrClosed,
context) {
protected override def read(): Array[Byte] = {
if (writer.exception.isDefined) {
diff --git
a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
index c9f83bca62cd..c9c5ffc9fa46 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
@@ -28,7 +28,6 @@ import javax.annotation.concurrent.GuardedBy
import scala.collection.mutable
import scala.jdk.CollectionConverters._
-import scala.jdk.OptionConverters._
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
@@ -117,13 +116,19 @@ private[spark] class PythonWorkerFactory(
private val authHelper = new SocketAuthHelper(conf)
private val isUnixDomainSock = authHelper.isUnixDomainSock
+ // The factory carries everything faulthandler-related through envVars
(which is how it reaches
+ // the worker process): PYTHON_FAULTHANDLER_DIR is set by the runner iff
faulthandler should be
+ // surfaced for this worker, so a worker handle created here binds that
directory (None = off).
+ private val faultHandlerLogDir: Option[File] =
+ envVars.get("PYTHON_FAULTHANDLER_DIR").map(new File(_))
+
@GuardedBy("self")
private var daemon: Process = null
val daemonHost = InetAddress.getLoopbackAddress()
@GuardedBy("self")
private var daemonPort: Int = 0
@GuardedBy("self")
- private val daemonWorkers = new mutable.WeakHashMap[PythonWorker,
ProcessHandle]()
+ private val daemonWorkers = new mutable.WeakHashMap[PythonWorker,
PythonWorkerHandle]()
@GuardedBy("self")
private var daemonSockPath: String = _
@GuardedBy("self")
@@ -144,7 +149,7 @@ private[spark] class PythonWorkerFactory(
envVars.getOrElse("PYTHONPATH", ""),
sys.env.getOrElse("PYTHONPATH", ""))
- def create(): (PythonWorker, Option[ProcessHandle]) = {
+ def create(): (PythonWorker, Option[PythonWorkerHandle]) = {
if (useDaemon) {
self.synchronized {
// Pull from idle workers until we get one that is alive, otherwise
create a new one.
@@ -180,9 +185,9 @@ private[spark] class PythonWorkerFactory(
* processes itself to avoid the high cost of forking from Java. This
currently only works
* on UNIX-based systems.
*/
- private def createThroughDaemon(): (PythonWorker, Option[ProcessHandle]) = {
+ private def createThroughDaemon(): (PythonWorker,
Option[PythonWorkerHandle]) = {
- def createWorker(): (PythonWorker, Option[ProcessHandle]) = {
+ def createWorker(): (PythonWorker, Option[PythonWorkerHandle]) = {
val socketChannel = if (isUnixDomainSock) {
SocketChannel.open(UnixDomainSocketAddress.of(daemonSockPath))
} else {
@@ -209,8 +214,8 @@ private[spark] class PythonWorkerFactory(
if (pid < 0) {
throw new IllegalStateException("Python daemon failed to launch worker
with code " + pid)
}
- val processHandle = ProcessHandle.of(pid).orElseThrow(
- () => new IllegalStateException("Python daemon failed to launch
worker.")
+ val processHandle = PythonWorkerHandle.of(pid,
faultHandlerLogDir).getOrElse(
+ throw new IllegalStateException("Python daemon failed to launch
worker.")
)
authHelper.authToServer(socketChannel)
socketChannel.configureBlocking(false)
@@ -247,7 +252,7 @@ private[spark] class PythonWorkerFactory(
* Launch a worker by executing worker.py (by default) directly and telling
it to connect to us.
*/
private[spark] def createSimpleWorker(
- blockingMode: Boolean): (PythonWorker, Option[ProcessHandle]) = {
+ blockingMode: Boolean): (PythonWorker, Option[PythonWorkerHandle]) = {
var serverSocketChannel: ServerSocketChannel = null
lazy val sockPath = new File(
authHelper.sockDir,
@@ -317,7 +322,7 @@ private[spark] class PythonWorkerFactory(
self.synchronized {
simpleWorkers.put(worker, workerProcess)
}
- (worker.refresh(), ProcessHandle.of(pid).toScala)
+ (worker.refresh(), PythonWorkerHandle.of(pid, faultHandlerLogDir))
} catch {
case e: Exception =>
throw new SparkException("Python worker failed to connect back.", e)
@@ -514,10 +519,11 @@ private[spark] class PythonWorkerFactory(
self.synchronized {
if (useDaemon) {
if (daemon != null) {
- daemonWorkers.get(worker).foreach { processHandle =>
+ daemonWorkers.get(worker).collect { case l: LocalPythonWorkerHandle
=> l.pid }
+ .foreach { pid =>
// tell daemon to kill worker by pid
val output = new DataOutputStream(daemon.getOutputStream)
- output.writeInt(processHandle.pid().toInt)
+ output.writeInt(pid.toInt)
output.flush()
daemon.getOutputStream.flush()
}
diff --git
a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerHandle.scala
b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerHandle.scala
new file mode 100644
index 000000000000..68d5928a1970
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerHandle.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.api.python
+
+import java.io.File
+import java.nio.file.{Files => JavaFiles}
+
+/**
+ * Abstracts the concrete process backend so the runner does not depend on the
local OS process
+ * directly, exposing only the operations it needs: liveness, termination, and
post-mortem
+ * diagnostics. [[LocalPythonWorkerHandle]] is the implementation backed by a
local OS process.
+ */
+trait PythonWorkerHandle {
+ def isAlive(): Boolean
+
+ /** Best-effort terminate the worker; true iff a termination request was
issued. */
+ def destroy(): Boolean
+
+ /**
+ * Best-effort post-mortem diagnostics for a terminated worker, if any: the
handle knows both how
+ * to locate its own worker's diagnostics and whether they are enabled
(bound when the handle was
+ * created). Empty when disabled or none are available.
+ */
+ def terminationDiagnostics(): Option[String]
+}
+
+/**
+ * Default [[PythonWorkerHandle]] backed by a local OS process.
+ *
+ * @param processHandle the JVM handle for the worker process
+ * @param faultHandlerLogDir the directory the worker's Python faulthandler
writes its log to, if
+ * faulthandler is enabled for this worker;
[[terminationDiagnostics]]
+ * reads `<dir>/<pid>`. None means faulthandler is
off, so no log is
+ * surfaced. The creator binds this once, since it
is fixed for the
+ * worker's whole lifetime.
+ */
+class LocalPythonWorkerHandle(
+ processHandle: ProcessHandle,
+ faultHandlerLogDir: Option[File] = None)
+ extends PythonWorkerHandle {
+
+ /** The OS pid of the worker process; an implementation detail of this local
backend. */
+ def pid: Long = processHandle.pid()
+
+ override def isAlive(): Boolean = processHandle.isAlive()
+
+ override def destroy(): Boolean = processHandle.destroy()
+
+ // The worker's faulthandler log lives at <faultHandlerLogDir>/<pid>; read
it once, deleting it so
+ // it is surfaced exactly once. None (faulthandler off) or a missing file
yields no diagnostics.
+ override def terminationDiagnostics(): Option[String] =
+ faultHandlerLogDir
+ .map(dir => new File(dir, pid.toString).toPath)
+ .collect {
+ case path if JavaFiles.exists(path) =>
+ val error = String.join("\n", JavaFiles.readAllLines(path)) + "\n"
+ JavaFiles.deleteIfExists(path)
+ error
+ }
+}
+
+object PythonWorkerHandle {
+
+ /**
+ * Create a [[LocalPythonWorkerHandle]] for the given pid, or None if the
pid is not valid.
+ *
+ * @param faultHandlerLogDir the directory the worker's faulthandler writes
to, or None if
+ * faulthandler is off for this worker. Bound onto
the handle and read
+ * by
[[LocalPythonWorkerHandle.terminationDiagnostics]].
+ */
+ def of(pid: Long, faultHandlerLogDir: Option[File] = None):
Option[PythonWorkerHandle] = {
+ val handle = ProcessHandle.of(pid)
+ if (handle.isPresent) {
+ Some(new LocalPythonWorkerHandle(handle.get, faultHandlerLogDir))
+ } else {
+ None
+ }
+ }
+}
diff --git
a/core/src/test/scala/org/apache/spark/api/python/PythonWorkerHandleSuite.scala
b/core/src/test/scala/org/apache/spark/api/python/PythonWorkerHandleSuite.scala
new file mode 100644
index 000000000000..14ec33c5c2c4
--- /dev/null
+++
b/core/src/test/scala/org/apache/spark/api/python/PythonWorkerHandleSuite.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.api.python
+
+import java.io.File
+import java.nio.file.Files
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.util.Utils
+
+class PythonWorkerHandleSuite extends SparkFunSuite {
+
+ test("terminationDiagnostics reads the worker's faulthandler log only when a
log dir is bound") {
+ // The handle reads its own worker's faulthandler log at
<faultHandlerLogDir>/<pid>; whether it
+ // is surfaced is a per-worker policy bound once at construction (None
means faulthandler off),
+ // so terminationDiagnostics takes no argument.
+ withTempDir { dir =>
+ val pid = ProcessHandle.current().pid()
+ Files.writeString(new File(dir, pid.toString).toPath, "py-traceback")
+ val disabled = new LocalPythonWorkerHandle(ProcessHandle.current(),
faultHandlerLogDir = None)
+ assert(disabled.terminationDiagnostics().isEmpty)
+ // Still present: the disabled handle above must not have consumed the
log.
+ val enabled =
+ new LocalPythonWorkerHandle(ProcessHandle.current(),
faultHandlerLogDir = Some(dir))
+
assert(enabled.terminationDiagnostics().exists(_.contains("py-traceback")))
+ }
+ }
+
+ test("terminationDiagnostics is empty when the worker wrote no faulthandler
log") {
+ withTempDir { dir =>
+ // Log dir bound, but no <pid> file was written.
+ val handle =
+ new LocalPythonWorkerHandle(ProcessHandle.current(),
faultHandlerLogDir = Some(dir))
+ assert(handle.terminationDiagnostics().isEmpty)
+ }
+ }
+
+ test("isAlive and destroy terminate the underlying process") {
+ assume(!Utils.isWindows)
+ val proc = new ProcessBuilder("sleep", "30").start()
+ try {
+ val handle = new LocalPythonWorkerHandle(proc.toHandle)
+ assert(handle.isAlive())
+ assert(handle.destroy()) // kill issued
+ assert(proc.waitFor(10, TimeUnit.SECONDS)) // process actually reaped
+ assert(!handle.isAlive())
+ } finally {
+ proc.destroyForcibly()
+ }
+ }
+
+ test("of returns a live handle that observes the process dying") {
+ assume(!Utils.isWindows)
+ val proc = new ProcessBuilder("sleep", "30").start()
+ try {
+ // Capture the handle while the process is alive; asserting liveness on
it after the kill
+ // avoids the pid-reuse race that re-querying of(reapedPid) would hit if
the OS recycles it.
+ val handle = PythonWorkerHandle.of(proc.pid())
+ assert(handle.isDefined)
+ assert(handle.get.isAlive())
+ proc.destroyForcibly()
+ assert(proc.waitFor(10, TimeUnit.SECONDS))
+ assert(!handle.get.isAlive())
+ } finally {
+ proc.destroyForcibly()
+ }
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
index bae1a2aa0d5d..c66e16f2f04c 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonArrowOutput.scala
@@ -25,7 +25,7 @@ import org.apache.arrow.vector.VectorSchemaRoot
import org.apache.arrow.vector.ipc.ArrowStreamReader
import org.apache.spark.{SparkEnv, TaskContext}
-import org.apache.spark.api.python.{BasePythonRunner, PythonWorker,
SpecialLengths}
+import org.apache.spark.api.python.{BasePythonRunner, PythonWorker,
PythonWorkerHandle, SpecialLengths}
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StructType
@@ -59,12 +59,12 @@ private[python] trait PythonArrowOutput[OUT <: AnyRef] {
self: BasePythonRunner[
startTime: Long,
env: SparkEnv,
worker: PythonWorker,
- pid: Option[Int],
+ handle: Option[PythonWorkerHandle],
releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[OUT] = {
new ReaderIterator(
- stream, writer, startTime, env, worker, pid, releasedOrClosed, context) {
+ stream, writer, startTime, env, worker, handle, releasedOrClosed,
context) {
private val allocator = ArrowUtils.rootAllocator.newChildAllocator(
s"stdin reader for $pythonExec", 0, Long.MaxValue)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
index 80887dd469d6..91f0b1d1203a 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala
@@ -28,7 +28,7 @@ import scala.jdk.CollectionConverters._
import net.razorvine.pickle.Pickler
import org.apache.spark.{JobArtifactSet, SparkEnv, SparkException}
-import org.apache.spark.api.python.{BasePythonRunner, PythonFunction,
PythonWorker, PythonWorkerException, PythonWorkerUtils, SpecialLengths}
+import org.apache.spark.api.python.{BasePythonRunner, PythonFunction,
PythonWorker, PythonWorkerException, PythonWorkerHandle, PythonWorkerUtils,
SpecialLengths}
import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.BUFFER_SIZE
import org.apache.spark.internal.config.Python._
@@ -116,9 +116,8 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
val pickler = new Pickler(/* useMemo = */ true,
/* valueCompare = */ false)
- val (worker: PythonWorker, handle: Option[ProcessHandle]) =
+ val (worker: PythonWorker, handle: Option[PythonWorkerHandle]) =
env.createPythonWorker(pythonExec, workerModule, envVars.asScala.toMap,
useDaemon)
- val pid = handle.map(_.pid.toInt)
var releasedOrClosed = false
val bufferStream = new DirectByteBufferOutputStream()
try {
@@ -137,7 +136,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
val dataIn = new DataInputStream(new BufferedInputStream(
new WorkerInputStream(
worker, bufferStream.toByteBuffer, handle,
- faultHandlerEnabled, idleTimeoutSeconds, killOnIdleTimeout),
+ idleTimeoutSeconds, killOnIdleTimeout),
bufferSize))
val res = receiveFromPython(dataIn)
@@ -164,7 +163,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
case e: IOException =>
val base = "Python worker exited unexpectedly (crashed)"
- val msg = tryReadFaultHandlerLog(faultHandlerEnabled, pid)
+ val msg = handle.flatMap(_.terminationDiagnostics())
.map(error => s"$base: $error")
.getOrElse(base)
throw new SparkException(msg, e)
@@ -191,8 +190,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
private class WorkerInputStream(
worker: PythonWorker,
buffer: ByteBuffer,
- handle: Option[ProcessHandle],
- faultHandlerEnabled: Boolean,
+ handle: Option[PythonWorkerHandle],
idleTimeoutSeconds: Long,
killOnIdleTimeout: Boolean) extends InputStream {
@@ -234,7 +232,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
pythonWorkerStatusMessageWithContext(handle, worker,
buffer.hasRemaining))
if (killOnIdleTimeout) {
handle.foreach { handle =>
- if (handle.isAlive) {
+ if (handle.isAlive()) {
logWarning(
log"Terminating Python planner worker process due to idle
timeout (timeout: " +
log"${MDC(LogKeys.PYTHON_WORKER_IDLE_TIMEOUT,
idleTimeoutSeconds)} seconds)")
@@ -262,7 +260,7 @@ abstract class PythonPlannerRunner[T](func: PythonFunction)
extends Logging {
if (n == -1 && pythonWorkerKilled) {
val base = "Python worker process terminated due to idle timeout " +
s"(timeout: $idleTimeoutSeconds seconds)"
- val msg = tryReadFaultHandlerLog(faultHandlerEnabled,
handle.map(_.pid.toInt))
+ val msg = handle.flatMap(_.terminationDiagnostics())
.map(error => s"$base: $error")
.getOrElse(base)
throw new PythonWorkerException(msg)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
index b2eaec1674b7..16b0aa918011 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
@@ -108,11 +108,11 @@ abstract class BasePythonUDFRunner(
startTime: Long,
env: SparkEnv,
worker: PythonWorker,
- pid: Option[Int],
+ handle: Option[PythonWorkerHandle],
releasedOrClosed: AtomicBoolean,
context: TaskContext): Iterator[Array[Byte]] = {
new ReaderIterator(
- stream, writer, startTime, env, worker, pid, releasedOrClosed, context) {
+ stream, writer, startTime, env, worker, handle, releasedOrClosed,
context) {
protected override def read(): Array[Byte] = {
if (writer.exception.isDefined) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]