This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 586f6008b [KYUUBI #6399] Spark Kyuubi UI supports both javax and
jakarta servlet namespaces
586f6008b is described below
commit 586f6008bda6aabec4e1de7c60609ab2caadbc59
Author: Cheng Pan <[email protected]>
AuthorDate: Wed May 22 13:03:06 2024 +0800
[KYUUBI #6399] Spark Kyuubi UI supports both javax and jakarta servlet
namespaces
# :mag: Description
Spark 4.0 migrated from `javax.servlet` to `jakarta.servlet` in
SPARK-47118, which breaks the binary compatibility of `SparkUITab` and
`WebUIPage` that Kyuubi used, thus breaking the previous assumption of Kyuubi
Spark SQL engine: single jar built with default Spark version, compatible with
all supported versions of Spark runtime.
## Describe Your Solution ๐ง
This PR uses bytebuddy to dynamically generate classes and Java reflection
find and dispatch method invocation in runtime, to recover the existing
compatibility of Kyuubi Spark SQL engine.
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
Build with Spark 3.5
```
build/dist --tgz --web-ui --spark-provided --flink-provided --hive-provided
-Pspark-3.5
```
It produces both Scala 2.12 and 2.13 Spark SQL engine jars
- `kyuubi-spark-sql-engine_2.12-1.10.0-SNAPSHOT.jar`
- `kyuubi-spark-sql-engine_2.13-1.10.0-SNAPSHOT.jar`
Run with Spark 3.4 Scala 2.12
<img width="1639" alt="image"
src="https://github.com/apache/kyuubi/assets/26535726/caeef30d-7467-4942-a56a-88a7c93ef7cc";>
Run with Spark 3.5 Scala 2.13
<img width="1639" alt="image"
src="https://github.com/apache/kyuubi/assets/26535726/c339c1e9-c07f-4952-9a57-098b832c889f";>
Run with Spark 4.0.0-preview1 Scala 2.13
<img width="1639" alt="image"
src="https://github.com/apache/kyuubi/assets/26535726/a3fb6e77-b27e-4634-8acf-245a26b39d2b";>
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6399 from pan3793/ui-4.0.
Closes #6399
e0104f6df [Cheng Pan] nit
a2f9df4fa [Cheng Pan] nit
c369ab2e3 [Cheng Pan] nit
ec1c45f66 [Cheng Pan] nit
3e05744d6 [Cheng Pan] fix
a7e38cc1e [Cheng Pan] nit
fa14a0d98 [Cheng Pan] refactor
9d0ce6111 [Cheng Pan] A work version
fc78b58e4 [Cheng Pan] fix startup
d74c1c0fe [Cheng Pan] fix
50066f563 [Cheng Pan] nit
f5ad4c760 [Cheng Pan] Kyuubi UI supports Spark 4.0
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
externals/kyuubi-spark-sql-engine/pom.xml | 10 ++
.../engine/spark/operation/SparkOperation.scala | 2 +-
.../engine/spark/session/SparkSessionImpl.scala | 2 +-
.../scala/org/apache/spark/ui/EnginePage.scala | 48 +++---
.../org/apache/spark/ui/EngineSessionPage.scala | 26 +++-
.../main/scala/org/apache/spark/ui/EngineTab.scala | 113 ++++++++++----
...lsHelper.scala => HttpServletRequestLike.scala} | 22 ++-
.../spark/ui/JakartaHttpServletRequest.scala | 171 +++++++++++++++++++++
.../apache/spark/ui/JavaxHttpServletRequest.scala | 170 ++++++++++++++++++++
.../scala/org/apache/spark/ui/SparkUIUtils.scala | 103 +++++++++++++
pom.xml | 7 +
11 files changed, 609 insertions(+), 65 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml
b/externals/kyuubi-spark-sql-engine/pom.xml
index 3155462a4..2d1dd433b 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -64,6 +64,11 @@
<artifactId>grpc-util</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ </dependency>
+
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@@ -233,6 +238,7 @@
<include>com.google.j2objc:j2objc-annotations</include>
<include>com.google.protobuf:*</include>
<include>dev.failsafe:failsafe</include>
+ <include>net.bytebuddy:byte-buddy</include>
<include>io.etcd:*</include>
<include>io.grpc:*</include>
<include>io.netty:*</include>
@@ -377,6 +383,10 @@
<pattern>com.google.type</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.type</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>net.bytebuddy</pattern>
+
<shadedPattern>${kyuubi.shade.packageName}.net.bytebuddy</shadedPattern>
+ </relocation>
</relocations>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
index d6d68da0c..b72447ae5 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
@@ -25,7 +25,7 @@ import org.apache.spark.kyuubi.SparkUtilsHelper.redact
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.sql.execution.SQLExecution
import org.apache.spark.sql.types.{BinaryType, StructField, StructType}
-import org.apache.spark.ui.SparkUIUtilsHelper.formatDuration
+import org.apache.spark.ui.SparkUIUtils.formatDuration
import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
index b0e2d91c5..85c918eb6 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.{AnalysisException, SparkSession}
-import org.apache.spark.ui.SparkUIUtilsHelper.formatDuration
+import org.apache.spark.ui.SparkUIUtils.formatDuration
import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
index d59b64dd9..36f01e316 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
@@ -20,7 +20,6 @@ package org.apache.spark.ui
import java.net.URLEncoder
import java.nio.charset.StandardCharsets.UTF_8
import java.util.Date
-import javax.servlet.http.HttpServletRequest
import scala.collection.JavaConverters.mapAsScalaMapConverter
import scala.collection.mutable
@@ -33,10 +32,21 @@ import org.apache.spark.ui.UIUtils._
import org.apache.kyuubi._
import org.apache.kyuubi.engine.spark.events.{SessionEvent,
SparkOperationEvent}
-case class EnginePage(parent: EngineTab) extends WebUIPage("") {
+abstract class EnginePage(parent: EngineTab) extends WebUIPage("") {
private val store = parent.store
- override def render(request: HttpServletRequest): Seq[Node] = {
+ def dispatchRender(req: AnyRef): Seq[Node] = req match {
+ case reqLike: HttpServletRequestLike =>
+ this.render0(reqLike)
+ case javaxReq: javax.servlet.http.HttpServletRequest =>
+ this.render0(HttpServletRequestLike.fromJavax(javaxReq))
+ case jakartaReq: jakarta.servlet.http.HttpServletRequest =>
+ this.render0(HttpServletRequestLike.fromJakarta(jakartaReq))
+ case unsupported =>
+ throw new IllegalArgumentException(s"Unsupported class
${unsupported.getClass.getName}")
+ }
+
+ def render0(request: HttpServletRequestLike): Seq[Node] = {
val onlineSession = new mutable.ArrayBuffer[SessionEvent]()
val closedSession = new mutable.ArrayBuffer[SessionEvent]()
@@ -77,7 +87,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
runningSqlStat.toSeq,
completedSqlStat.toSeq,
failedSqlStat.toSeq)
- UIUtils.headerSparkPage(request, parent.name, content, parent)
+ SparkUIUtils.headerSparkPage(request, parent.name, content, parent)
}
private def generateBasicStats(): Seq[Node] = {
@@ -131,8 +141,8 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
</ul>
}
- private def stop(request: HttpServletRequest): Seq[Node] = {
- val basePath = UIUtils.prependBaseUri(request, parent.basePath)
+ private def stop(request: HttpServletRequestLike): Seq[Node] = {
+ val basePath = SparkUIUtils.prependBaseUri(request, parent.basePath)
if (parent.killEnabled) {
val confirmForceStop =
s"if (window.confirm('Are you sure you want to stop kyuubi engine
immediately ?')) " +
@@ -160,7 +170,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
/** Generate stats of running statements for the engine */
private def generateStatementStatsTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
running: Seq[SparkOperationEvent],
completed: Seq[SparkOperationEvent],
failed: Seq[SparkOperationEvent]): Seq[Node] = {
@@ -241,7 +251,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
}
private def statementStatsTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
sqlTableTag: String,
parent: EngineTab,
data: Seq[SparkOperationEvent]): Seq[Node] = {
@@ -255,7 +265,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
parent,
data,
"kyuubi",
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
s"${sqlTableTag}-table").table(sqlTablePage)
} catch {
case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
@@ -270,7 +280,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
/** Generate stats of online sessions for the engine */
private def generateSessionStatsTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
online: Seq[SessionEvent],
closed: Seq[SessionEvent]): Seq[Node] = {
val content = mutable.ListBuffer[Node]()
@@ -326,7 +336,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
}
private def sessionTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
sessionTage: String,
parent: EngineTab,
data: Seq[SessionEvent]): Seq[Node] = {
@@ -338,7 +348,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
parent,
data,
"kyuubi",
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
s"${sessionTage}-table").table(sessionPage)
} catch {
case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
@@ -352,7 +362,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
}
private class SessionStatsPagedTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
parent: EngineTab,
data: Seq[SessionEvent],
subPath: String,
@@ -418,7 +428,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
override def row(session: SessionEvent): Seq[Node] = {
val sessionLink = "%s/%s/session/?id=%s".format(
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
parent.prefix,
session.sessionId)
<tr>
@@ -440,7 +450,7 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
}
private class StatementStatsPagedTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
parent: EngineTab,
data: Seq[SparkOperationEvent],
subPath: String,
@@ -507,7 +517,7 @@ private class StatementStatsPagedTable(
override def row(event: SparkOperationEvent): Seq[Node] = {
val sessionLink = "%s/%s/session/?id=%s".format(
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
parent.prefix,
event.sessionId)
<tr>
@@ -544,7 +554,7 @@ private class StatementStatsPagedTable(
if (event.executionId.isDefined) {
<a href={
"%s/SQL/execution/?id=%s".format(
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
event.executionId.get)
}>
{event.executionId.get}
@@ -659,7 +669,7 @@ private object TableSourceUtil {
* Returns parameter of this table.
*/
def getRequestTableParameters(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
tableTag: String,
defaultSortColumn: String): (String, Boolean, Int) = {
val parameterSortColumn = request.getParameter(s"$tableTag.sort")
@@ -678,7 +688,7 @@ private object TableSourceUtil {
/**
* Returns parameters of other tables in the page.
*/
- def getRequestParameterOtherTable(request: HttpServletRequest, tableTag:
String): String = {
+ def getRequestParameterOtherTable(request: HttpServletRequestLike, tableTag:
String): String = {
request.getParameterMap.asScala
.filterNot(_._1.startsWith(tableTag))
.map(parameter => parameter._1 + "=" + parameter._2(0))
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
index 46011ceae..a6aae725a 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala
@@ -18,7 +18,6 @@
package org.apache.spark.ui
import java.util.Date
-import javax.servlet.http.HttpServletRequest
import scala.collection.mutable
import scala.xml.Node
@@ -31,7 +30,7 @@ import org.apache.spark.util.Utils
import org.apache.kyuubi.engine.spark.events.SparkOperationEvent
/** Page for Spark Web UI that shows statistics of jobs running in the engine
server */
-case class EngineSessionPage(parent: EngineTab)
+abstract class EngineSessionPage(parent: EngineTab)
extends WebUIPage("session") with Logging {
val store = parent.store
@@ -39,8 +38,19 @@ case class EngineSessionPage(parent: EngineTab)
private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha")
private def propertyRow(kv: (String, String)) =
<tr><td>{kv._1}</td><td>{kv._2}</td></tr>
+ def dispatchRender(req: AnyRef): Seq[Node] = req match {
+ case reqLike: HttpServletRequestLike =>
+ this.render0(reqLike)
+ case javaxReq: javax.servlet.http.HttpServletRequest =>
+ this.render0(HttpServletRequestLike.fromJavax(javaxReq))
+ case jakartaReq: jakarta.servlet.http.HttpServletRequest =>
+ this.render0(HttpServletRequestLike.fromJakarta(jakartaReq))
+ case unsupported =>
+ throw new IllegalArgumentException(s"Unsupported class
${unsupported.getClass.getName}")
+ }
+
/** Render the page */
- def render(request: HttpServletRequest): Seq[Node] = {
+ def render0(request: HttpServletRequestLike): Seq[Node] = {
val parameterId = request.getParameter("id")
require(parameterId != null && parameterId.nonEmpty, "Missing id
parameter")
@@ -98,7 +108,7 @@ case class EngineSessionPage(parent: EngineTab)
sessionPropertiesTable ++
generateSQLStatsTable(request, sessionStat.sessionId)
}
- UIUtils.headerSparkPage(request, parent.name + " Session", content, parent)
+ SparkUIUtils.headerSparkPage(request, parent.name + " Session", content,
parent)
}
/** Generate basic stats of the engine server */
@@ -128,7 +138,9 @@ case class EngineSessionPage(parent: EngineTab)
}
/** Generate stats of batch statements of the engine server */
- private def generateSQLStatsTable(request: HttpServletRequest, sessionID:
String): Seq[Node] = {
+ private def generateSQLStatsTable(
+ request: HttpServletRequestLike,
+ sessionID: String): Seq[Node] = {
val running = new mutable.ArrayBuffer[SparkOperationEvent]()
val completed = new mutable.ArrayBuffer[SparkOperationEvent]()
val failed = new mutable.ArrayBuffer[SparkOperationEvent]()
@@ -218,7 +230,7 @@ case class EngineSessionPage(parent: EngineTab)
}
private def statementStatsTable(
- request: HttpServletRequest,
+ request: HttpServletRequestLike,
sqlTableTag: String,
parent: EngineTab,
data: Seq[SparkOperationEvent]): Seq[Node] = {
@@ -231,7 +243,7 @@ case class EngineSessionPage(parent: EngineTab)
parent,
data,
"kyuubi/session",
- UIUtils.prependBaseUri(request, parent.basePath),
+ SparkUIUtils.prependBaseUri(request, parent.basePath),
s"${sqlTableTag}").table(sqlTablePage)
} catch {
case e @ (_: IllegalArgumentException | _: IndexOutOfBoundsException) =>
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala
index 52edcf220..5c031f943 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala
@@ -17,12 +17,18 @@
package org.apache.spark.ui
-import javax.servlet.http.HttpServletRequest
-
import scala.util.control.NonFatal
+import net.bytebuddy.ByteBuddy
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy
+import net.bytebuddy.dynamic.scaffold.subclass.ConstructorStrategy
+import net.bytebuddy.implementation.MethodCall
+import net.bytebuddy.matcher.ElementMatchers.{isConstructor, named}
+import org.apache.spark.util.{Utils => SparkUtils}
+
import org.apache.kyuubi.{Logging, Utils}
import org.apache.kyuubi.config.KyuubiConf
+import
org.apache.kyuubi.engine.spark.KyuubiSparkUtil.SPARK_ENGINE_RUNTIME_VERSION
import org.apache.kyuubi.engine.spark.SparkSQLEngine
import org.apache.kyuubi.engine.spark.events.EngineEventsStore
import org.apache.kyuubi.service.ServiceState
@@ -53,49 +59,96 @@ case class EngineTab(
.getOrElse(0L)
}
+ private val enginePage = {
+ val dispatchMethod = classOf[EnginePage].getMethod("dispatchRender",
classOf[AnyRef])
+ new ByteBuddy()
+ .subclass(classOf[EnginePage],
ConstructorStrategy.Default.IMITATE_SUPER_CLASS_PUBLIC)
+ .method(isConstructor()).intercept(MethodCall.invokeSuper())
+
.method(named("render")).intercept(MethodCall.invoke(dispatchMethod).withAllArguments())
+ .make()
+ .load(SparkUtils.getContextOrSparkClassLoader,
ClassLoadingStrategy.Default.INJECTION)
+ .getLoaded
+ .getDeclaredConstructor(classOf[EngineTab])
+ .newInstance(this)
+ }
+
+ private val engineSessionPage = {
+ val dispatchMethod =
classOf[EngineSessionPage].getMethod("dispatchRender", classOf[AnyRef])
+ new ByteBuddy()
+ .subclass(classOf[EngineSessionPage],
ConstructorStrategy.Default.IMITATE_SUPER_CLASS_PUBLIC)
+ .method(isConstructor()).intercept(MethodCall.invokeSuper())
+
.method(named("render")).intercept(MethodCall.invoke(dispatchMethod).withAllArguments())
+ .make()
+ .load(SparkUtils.getContextOrSparkClassLoader,
ClassLoadingStrategy.Default.INJECTION)
+ .getLoaded
+ .getDeclaredConstructor(classOf[EngineTab])
+ .newInstance(this)
+ }
+
sparkUI.foreach { ui =>
- this.attachPage(EnginePage(this))
- this.attachPage(EngineSessionPage(this))
+ this.attachPage(enginePage)
+ this.attachPage(engineSessionPage)
ui.attachTab(this)
Utils.addShutdownHook(() => ui.detachTab(this))
}
sparkUI.foreach { ui =>
try {
- // [KYUUBI #3627]: the official spark release uses the shaded and
relocated jetty classes,
- // but if we use sbt to build for testing, e.g. docker image, it still
uses the vanilla
- // jetty classes.
val sparkServletContextHandlerClz = DynClasses.builder()
+ // for official Spark releases and distributions built via Maven
.impl("org.sparkproject.jetty.servlet.ServletContextHandler")
+ // for distributions built via SBT
.impl("org.eclipse.jetty.servlet.ServletContextHandler")
.buildChecked()
val attachHandlerMethod = DynMethods.builder("attachHandler")
.impl("org.apache.spark.ui.SparkUI", sparkServletContextHandlerClz)
.buildChecked(ui)
- val createRedirectHandlerMethod =
DynMethods.builder("createRedirectHandler")
- .impl(
- "org.apache.spark.ui.JettyUtils",
- classOf[String],
- classOf[String],
- classOf[HttpServletRequest => Unit],
- classOf[String],
- classOf[Set[String]])
- .buildStaticChecked()
-
- attachHandlerMethod
- .invoke(
+
+ if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
+ attachHandlerMethod.invoke {
+ val createRedirectHandlerMethod =
DynMethods.builder("createRedirectHandler")
+ .impl(
+ JettyUtils.getClass,
+ classOf[String],
+ classOf[String],
+ classOf[jakarta.servlet.http.HttpServletRequest => Unit],
+ classOf[String],
+ classOf[Set[String]])
+ .buildChecked(JettyUtils)
+
+ val killHandler =
+ (_: jakarta.servlet.http.HttpServletRequest) => handleKill()
+ val gracefulKillHandler =
+ (_: jakarta.servlet.http.HttpServletRequest) =>
handleGracefulKill()
+
createRedirectHandlerMethod
- .invoke("/kyuubi/stop", "/kyuubi", handleKillRequest _, "",
Set("GET", "POST")))
+ .invoke("/kyuubi/stop", "/kyuubi", killHandler, "", Set("GET",
"POST"))
+ createRedirectHandlerMethod
+ .invoke("/kyuubi/gracefulstop", "/kyuubi", gracefulKillHandler,
"", Set("GET", "POST"))
+ }
+ } else {
+ val createRedirectHandlerMethod =
DynMethods.builder("createRedirectHandler")
+ .impl(
+ JettyUtils.getClass,
+ classOf[String],
+ classOf[String],
+ classOf[javax.servlet.http.HttpServletRequest => Unit],
+ classOf[String],
+ classOf[Set[String]])
+ .buildChecked(JettyUtils)
+
+ attachHandlerMethod.invoke {
+ val killHandler =
+ (_: javax.servlet.http.HttpServletRequest) => handleKill()
+ val gracefulKillHandler =
+ (_: javax.servlet.http.HttpServletRequest) => handleGracefulKill()
- attachHandlerMethod
- .invoke(
createRedirectHandlerMethod
- .invoke(
- "/kyuubi/gracefulstop",
- "/kyuubi",
- handleGracefulKillRequest _,
- "",
- Set("GET", "POST")))
+ .invoke("/kyuubi/stop", "/kyuubi", killHandler, "", Set("GET",
"POST"))
+ createRedirectHandlerMethod
+ .invoke("/kyuubi/gracefulstop", "/kyuubi", gracefulKillHandler,
"", Set("GET", "POST"))
+ }
+ }
} catch {
case NonFatal(cause) => reportInstallError(cause)
case cause: NoClassDefFoundError => reportInstallError(cause)
@@ -109,13 +162,13 @@ case class EngineTab(
cause)
}
- def handleKillRequest(request: HttpServletRequest): Unit = {
+ def handleKill(): Unit = {
if (killEnabled && engine.isDefined && engine.get.getServiceState !=
ServiceState.STOPPED) {
engine.get.stop()
}
}
- def handleGracefulKillRequest(request: HttpServletRequest): Unit = {
+ def handleGracefulKill(): Unit = {
if (killEnabled && engine.isDefined && engine.get.getServiceState !=
ServiceState.STOPPED) {
engine.get.gracefulStop()
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtilsHelper.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/HttpServletRequestLike.scala
similarity index 65%
rename from
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtilsHelper.scala
rename to
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/HttpServletRequestLike.scala
index c60fe4466..7b66cb87b 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtilsHelper.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/HttpServletRequestLike.scala
@@ -17,13 +17,21 @@
package org.apache.spark.ui
-/**
- * A place to invoke non-public APIs of [[UIUtils]], anything to be added here
need to
- * think twice
- */
-object SparkUIUtilsHelper {
+import java.util
+
+trait HttpServletRequestLike {
+
+ def getParameter(name: String): String
+
+ def getParameterMap: util.Map[String, Array[String]]
+}
+
+object HttpServletRequestLike {
+ def fromJavax(req: javax.servlet.http.HttpServletRequest):
JavaxHttpServletRequest = {
+ new JavaxHttpServletRequest(req)
+ }
- def formatDuration(ms: Long): String = {
- UIUtils.formatDuration(ms)
+ def fromJakarta(req: jakarta.servlet.http.HttpServletRequest):
JakartaHttpServletRequest = {
+ new JakartaHttpServletRequest(req)
}
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JakartaHttpServletRequest.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JakartaHttpServletRequest.scala
new file mode 100644
index 000000000..9219f294a
--- /dev/null
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JakartaHttpServletRequest.scala
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import java.io.BufferedReader
+import java.security.Principal
+import java.util
+import java.util.Locale
+
+import jakarta.servlet._
+import jakarta.servlet.http._
+
+class JakartaHttpServletRequest(req: HttpServletRequest)
+ extends HttpServletRequest with HttpServletRequestLike {
+ override def getAuthType: String = req.getAuthType
+
+ override def getCookies: Array[Cookie] = req.getCookies
+
+ override def getDateHeader(s: String): Long = req.getDateHeader(s)
+
+ override def getHeader(s: String): String = req.getHeader(s)
+
+ override def getHeaders(s: String): util.Enumeration[String] =
req.getHeaders(s)
+
+ override def getHeaderNames: util.Enumeration[String] = req.getHeaderNames
+
+ override def getIntHeader(s: String): Int = req.getIntHeader(s)
+
+ override def getMethod: String = req.getMethod
+
+ override def getPathInfo: String = req.getPathInfo
+
+ override def getPathTranslated: String = req.getPathTranslated
+
+ override def getContextPath: String = req.getContextPath
+
+ override def getQueryString: String = req.getQueryString
+
+ override def getRemoteUser: String = req.getRemoteUser
+
+ override def isUserInRole(s: String): Boolean = req.isUserInRole(s)
+
+ override def getUserPrincipal: Principal = req.getUserPrincipal
+
+ override def getRequestedSessionId: String = req.getRequestedSessionId
+
+ override def getRequestURI: String = req.getRequestURI
+
+ override def getRequestURL: StringBuffer = req.getRequestURL
+
+ override def getServletPath: String = req.getServletPath
+
+ override def getSession(b: Boolean): HttpSession = req.getSession(b)
+
+ override def getSession: HttpSession = req.getSession
+
+ override def changeSessionId(): String = req.changeSessionId()
+
+ override def isRequestedSessionIdValid: Boolean =
req.isRequestedSessionIdValid
+
+ override def isRequestedSessionIdFromCookie: Boolean =
req.isRequestedSessionIdFromCookie
+
+ override def isRequestedSessionIdFromURL: Boolean =
req.isRequestedSessionIdFromURL
+
+ override def isRequestedSessionIdFromUrl: Boolean =
req.isRequestedSessionIdFromUrl
+
+ override def authenticate(httpServletResponse: HttpServletResponse): Boolean
=
+ req.authenticate(httpServletResponse)
+
+ override def login(s: String, s1: String): Unit = req.login(s, s1)
+
+ override def logout(): Unit = req.logout()
+
+ override def getParts: util.Collection[Part] = req.getParts
+
+ override def getPart(s: String): Part = req.getPart(s)
+
+ override def upgrade[T <: HttpUpgradeHandler](aClass: Class[T]): T =
req.upgrade(aClass)
+
+ override def getAttribute(s: String): AnyRef = req.getAttribute(s)
+
+ override def getAttributeNames: util.Enumeration[String] =
req.getAttributeNames
+
+ override def getCharacterEncoding: String = req.getCharacterEncoding
+
+ override def setCharacterEncoding(s: String): Unit =
req.setCharacterEncoding(s)
+
+ override def getContentLength: Int = req.getContentLength
+
+ override def getContentLengthLong: Long = req.getContentLengthLong
+
+ override def getContentType: String = req.getContentType
+
+ override def getInputStream: ServletInputStream = req.getInputStream
+
+ override def getParameter(s: String): String = req.getParameter(s)
+
+ override def getParameterNames: util.Enumeration[String] =
req.getParameterNames
+
+ override def getParameterValues(s: String): Array[String] =
req.getParameterValues(s)
+
+ override def getParameterMap: util.Map[String, Array[String]] =
req.getParameterMap
+
+ override def getProtocol: String = req.getProtocol
+
+ override def getScheme: String = req.getScheme
+
+ override def getServerName: String = req.getServerName
+
+ override def getServerPort: Int = req.getServerPort
+
+ override def getReader: BufferedReader = req.getReader
+
+ override def getRemoteAddr: String = req.getRemoteAddr
+
+ override def getRemoteHost: String = req.getRemoteHost
+
+ override def setAttribute(s: String, o: Any): Unit = req.setAttribute(s, o)
+
+ override def removeAttribute(s: String): Unit = req.removeAttribute(s)
+
+ override def getLocale: Locale = req.getLocale
+
+ override def getLocales: util.Enumeration[Locale] = req.getLocales
+
+ override def isSecure: Boolean = req.isSecure
+
+ override def getRequestDispatcher(s: String): RequestDispatcher =
req.getRequestDispatcher(s)
+
+ override def getRealPath(s: String): String = req.getRealPath(s)
+
+ override def getRemotePort: Int = req.getRemotePort
+
+ override def getLocalName: String = req.getLocalName
+
+ override def getLocalAddr: String = req.getLocalAddr
+
+ override def getLocalPort: Int = req.getLocalPort
+
+ override def getServletContext: ServletContext = req.getServletContext
+
+ override def startAsync(): AsyncContext = req.startAsync()
+
+ override def startAsync(
+ servletRequest: ServletRequest,
+ servletResponse: ServletResponse): AsyncContext =
+ req.startAsync(servletRequest, servletResponse)
+
+ override def isAsyncStarted: Boolean = req.isAsyncStarted
+
+ override def isAsyncSupported: Boolean = req.isAsyncSupported
+
+ override def getAsyncContext: AsyncContext = req.getAsyncContext
+
+ override def getDispatcherType: DispatcherType = req.getDispatcherType
+}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JavaxHttpServletRequest.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JavaxHttpServletRequest.scala
new file mode 100644
index 000000000..d77948001
--- /dev/null
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/JavaxHttpServletRequest.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import java.io.BufferedReader
+import java.security.Principal
+import java.util
+import java.util.Locale
+import javax.servlet._
+import javax.servlet.http._
+
+class JavaxHttpServletRequest(req: HttpServletRequest)
+ extends HttpServletRequest with HttpServletRequestLike {
+ override def getAuthType: String = req.getAuthType
+
+ override def getCookies: Array[Cookie] = req.getCookies
+
+ override def getDateHeader(s: String): Long = req.getDateHeader(s)
+
+ override def getHeader(s: String): String = req.getHeader(s)
+
+ override def getHeaders(s: String): util.Enumeration[String] =
req.getHeaders(s)
+
+ override def getHeaderNames: util.Enumeration[String] = req.getHeaderNames
+
+ override def getIntHeader(s: String): Int = req.getIntHeader(s)
+
+ override def getMethod: String = req.getMethod
+
+ override def getPathInfo: String = req.getPathInfo
+
+ override def getPathTranslated: String = req.getPathTranslated
+
+ override def getContextPath: String = req.getContextPath
+
+ override def getQueryString: String = req.getQueryString
+
+ override def getRemoteUser: String = req.getRemoteUser
+
+ override def isUserInRole(s: String): Boolean = req.isUserInRole(s)
+
+ override def getUserPrincipal: Principal = req.getUserPrincipal
+
+ override def getRequestedSessionId: String = req.getRequestedSessionId
+
+ override def getRequestURI: String = req.getRequestURI
+
+ override def getRequestURL: StringBuffer = req.getRequestURL
+
+ override def getServletPath: String = req.getServletPath
+
+ override def getSession(b: Boolean): HttpSession = req.getSession(b)
+
+ override def getSession: HttpSession = req.getSession
+
+ override def changeSessionId(): String = req.changeSessionId()
+
+ override def isRequestedSessionIdValid: Boolean =
req.isRequestedSessionIdValid
+
+ override def isRequestedSessionIdFromCookie: Boolean =
req.isRequestedSessionIdFromCookie
+
+ override def isRequestedSessionIdFromURL: Boolean =
req.isRequestedSessionIdFromURL
+
+ override def isRequestedSessionIdFromUrl: Boolean =
req.isRequestedSessionIdFromUrl
+
+ override def authenticate(httpServletResponse: HttpServletResponse): Boolean
=
+ req.authenticate(httpServletResponse)
+
+ override def login(s: String, s1: String): Unit = req.login(s, s1)
+
+ override def logout(): Unit = req.logout()
+
+ override def getParts: util.Collection[Part] = req.getParts
+
+ override def getPart(s: String): Part = req.getPart(s)
+
+ override def upgrade[T <: HttpUpgradeHandler](aClass: Class[T]): T =
req.upgrade(aClass)
+
+ override def getAttribute(s: String): AnyRef = req.getAttribute(s)
+
+ override def getAttributeNames: util.Enumeration[String] =
req.getAttributeNames
+
+ override def getCharacterEncoding: String = req.getCharacterEncoding
+
+ override def setCharacterEncoding(s: String): Unit =
req.setCharacterEncoding(s)
+
+ override def getContentLength: Int = req.getContentLength
+
+ override def getContentLengthLong: Long = req.getContentLengthLong
+
+ override def getContentType: String = req.getContentType
+
+ override def getInputStream: ServletInputStream = req.getInputStream
+
+ override def getParameter(s: String): String = req.getParameter(s)
+
+ override def getParameterNames: util.Enumeration[String] =
req.getParameterNames
+
+ override def getParameterValues(s: String): Array[String] =
req.getParameterValues(s)
+
+ override def getParameterMap: util.Map[String, Array[String]] =
req.getParameterMap
+
+ override def getProtocol: String = req.getProtocol
+
+ override def getScheme: String = req.getScheme
+
+ override def getServerName: String = req.getServerName
+
+ override def getServerPort: Int = req.getServerPort
+
+ override def getReader: BufferedReader = req.getReader
+
+ override def getRemoteAddr: String = req.getRemoteAddr
+
+ override def getRemoteHost: String = req.getRemoteHost
+
+ override def setAttribute(s: String, o: Any): Unit = req.setAttribute(s, o)
+
+ override def removeAttribute(s: String): Unit = req.removeAttribute(s)
+
+ override def getLocale: Locale = req.getLocale
+
+ override def getLocales: util.Enumeration[Locale] = req.getLocales
+
+ override def isSecure: Boolean = req.isSecure
+
+ override def getRequestDispatcher(s: String): RequestDispatcher =
req.getRequestDispatcher(s)
+
+ override def getRealPath(s: String): String = req.getRealPath(s)
+
+ override def getRemotePort: Int = req.getRemotePort
+
+ override def getLocalName: String = req.getLocalName
+
+ override def getLocalAddr: String = req.getLocalAddr
+
+ override def getLocalPort: Int = req.getLocalPort
+
+ override def getServletContext: ServletContext = req.getServletContext
+
+ override def startAsync(): AsyncContext = req.startAsync()
+
+ override def startAsync(
+ servletRequest: ServletRequest,
+ servletResponse: ServletResponse): AsyncContext =
+ req.startAsync(servletRequest, servletResponse)
+
+ override def isAsyncStarted: Boolean = req.isAsyncStarted
+
+ override def isAsyncSupported: Boolean = req.isAsyncSupported
+
+ override def getAsyncContext: AsyncContext = req.getAsyncContext
+
+ override def getDispatcherType: DispatcherType = req.getDispatcherType
+}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
new file mode 100644
index 000000000..4862372a1
--- /dev/null
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import java.lang.{Boolean => JBoolean}
+
+import scala.xml.Node
+
+import
org.apache.kyuubi.engine.spark.KyuubiSparkUtil.SPARK_ENGINE_RUNTIME_VERSION
+import org.apache.kyuubi.util.reflect.DynMethods
+
+/**
+ * A place to invoke non-public APIs of [[UIUtils]], anything to be added here
need to
+ * think twice
+ */
+object SparkUIUtils {
+
+ def formatDuration(ms: Long): String = {
+ UIUtils.formatDuration(ms)
+ }
+
+ def headerSparkPage(
+ request: HttpServletRequestLike,
+ title: String,
+ content: Seq[Node],
+ activeTab: SparkUITab,
+ helpText: Option[String] = None,
+ showVisualization: JBoolean = false,
+ useDataTables: JBoolean = false): Seq[Node] = {
+ val headerSparkPageMethod = if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
+ DynMethods.builder("headerSparkPage")
+ .impl(
+ UIUtils.getClass,
+ classOf[jakarta.servlet.http.HttpServletRequest],
+ classOf[String],
+ classOf[() => Seq[Node]],
+ classOf[SparkUITab],
+ classOf[Option[String]],
+ classOf[Boolean],
+ classOf[Boolean])
+ .buildChecked(UIUtils)
+ } else {
+ DynMethods.builder("headerSparkPage")
+ .impl(
+ UIUtils.getClass,
+ classOf[javax.servlet.http.HttpServletRequest],
+ classOf[String],
+ classOf[() => Seq[Node]],
+ classOf[SparkUITab],
+ classOf[Option[String]],
+ classOf[Boolean],
+ classOf[Boolean])
+ .buildChecked(UIUtils)
+ }
+ headerSparkPageMethod.invoke[Seq[Node]](
+ request,
+ title,
+ () => content,
+ activeTab,
+ helpText,
+ showVisualization,
+ useDataTables)
+ }
+
+ def prependBaseUri(
+ request: HttpServletRequestLike,
+ basePath: String = "",
+ resource: String = ""): String = {
+ val prependBaseUriMethod = if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
+ DynMethods.builder("prependBaseUri")
+ .impl(
+ UIUtils.getClass,
+ classOf[jakarta.servlet.http.HttpServletRequest],
+ classOf[String],
+ classOf[String])
+ .buildChecked(UIUtils)
+ } else {
+ DynMethods.builder("prependBaseUri")
+ .impl(
+ UIUtils.getClass,
+ classOf[javax.servlet.http.HttpServletRequest],
+ classOf[String],
+ classOf[String])
+ .buildChecked(UIUtils)
+ }
+ prependBaseUriMethod.invoke[String](request, basePath, resource)
+ }
+}
diff --git a/pom.xml b/pom.xml
index d83187d65..513271d0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,6 +125,7 @@
<antlr.st4.version>4.3.4</antlr.st4.version>
<apache.archive.dist>https://archive.apache.org/dist</apache.archive.dist>
<atlas.version>2.3.0</atlas.version>
+ <byte-buddy.version>1.14.15</byte-buddy.version>
<bouncycastle.version>1.78</bouncycastle.version>
<codahale.metrics.version>4.2.23</codahale.metrics.version>
<commons-cli.version>1.5.0</commons-cli.version>
@@ -838,6 +839,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <version>${byte-buddy.version}</version>
+ </dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
