Github user gengliangwang commented on a diff in the pull request:
https://github.com/apache/spark/pull/18499#discussion_r131118306
--- Diff: core/src/main/scala/org/apache/spark/ui/JettyUtils.scala ---
@@ -194,30 +194,26 @@ private[spark] object JettyUtils extends Logging {
}
/** Create a handler for proxying request to Workers and Application
Drivers */
- def createProxyHandler(
- prefix: String,
- target: String): ServletContextHandler = {
+ def createProxyHandler(idToUiAddress: String => Option[String]):
ServletContextHandler = {
val servlet = new ProxyServlet {
override def rewriteTarget(request: HttpServletRequest): String = {
- val rewrittenURI = createProxyURI(
- prefix, target, request.getRequestURI(),
request.getQueryString())
- if (rewrittenURI == null) {
- return null
- }
- if (!validateDestination(rewrittenURI.getHost(),
rewrittenURI.getPort())) {
- return null
- }
- rewrittenURI.toString()
+ val path = request.getPathInfo
+ if (path == null) return null
+
+ val prefixTrailingSlashIndex = path.indexOf('/', 1)
+ val prefix = path.substring(0,
+ if (prefixTrailingSlashIndex == -1) path.length else
prefixTrailingSlashIndex)
+ val id = prefix.drop(1)
+
+ idToUiAddress(id)
+ .map(createProxyURI(prefix, _, path, request.getQueryString))
+ .filter(uri => uri != null && validateDestination(uri.getHost,
uri.getPort))
+ .map(_.toString)
+ .orNull
}
override def newHttpClient(): HttpClient = {
- // SPARK-21176: Use the Jetty logic to calculate the number of
selector threads (#CPUs/2),
- // but limit it to 8 max.
- // Otherwise, it might happen that we exhaust the threadpool since
in reverse proxy mode
- // a proxy is instantiated for each executor. If the head node has
many processors, this
- // can quickly add up to an unreasonably high number of threads.
- val numSelectors = math.max(1, math.min(8,
Runtime.getRuntime().availableProcessors() / 2))
- new HttpClient(new HttpClientTransportOverHTTP(numSelectors), null)
+ new HttpClient(new HttpClientTransportOverHTTP(8), null)
--- End diff --
I think we can just keep the previous code here. If the number of available
processors is smaller than 8, e.g. only one processor, we don't need to make
it 8 threads.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
