hvanhovell commented on code in PR #55768:
URL: https://github.com/apache/spark/pull/55768#discussion_r3282058564
##########
core/src/main/scala/org/apache/spark/SparkEnv.scala:
##########
@@ -120,6 +123,48 @@ class SparkEnv (
pythonExec: String, workerModule: String, daemonModule: String, envVars:
Map[String, String])
private val pythonWorkers = mutable.HashMap[PythonWorkersKey,
PythonWorkerFactory]()
+ /**
+ * :: Experimental ::
+ * Dispatcher factory to generate UDF worker dispatchers
+ * using the new UDF framework proposed in SPARK-55278.
+ * Initialized on first use via [[getExternalUDFDispatcher]].
+ */
+ @volatile private var udfDispatcherManager: Option[UDFDispatcherManager] =
None
+
+ private def createUDFDispatcherManager(): UDFDispatcherManager = {
+ val factory = new UDFDispatcherFactory {
+ override def createDispatcher(
+ workerSpec: UDFWorkerSpecification,
+ logger: org.apache.spark.udf.worker.core.WorkerLogger
+ ): WorkerDispatcher = {
+ // TODO [SPARK-55278]: Wire in the correct dispatcher factory
+ throw new UnsupportedOperationException(
+ "No UDF dispatcher factory configured. " +
+ "Set up a concrete factory for SPARK-55278.")
+ }
+ }
+ new UDFDispatcherManager(factory, new SparkUDFWorkerLogger())
+ }
+
+ /**
+ * :: Experimental ::
+ * Returns the [[WorkerDispatcher]] for the given worker
+ * specification via the [[UDFDispatcherManager]].
+ */
+ private[spark] def getExternalUDFDispatcher(
+ workerSpec: UDFWorkerSpecification): WorkerDispatcher = {
+ val manager : UDFDispatcherManager = udfDispatcherManager.getOrElse {
+ synchronized {
+ // Get or Else synchronized to protect
+ // against concurrent creation requests.
+ udfDispatcherManager.getOrElse {
Review Comment:
Clever...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
