This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 73c84d4 [KYUUBI #1868] Upgrate netty from 4.1.68 to 4.1.73
73c84d4 is described below
commit 73c84d42e7509bf683f23df903493941e2b14d63
Author: ulysses-you <[email protected]>
AuthorDate: Wed Feb 9 10:57:46 2022 +0800
[KYUUBI #1868] Upgrate netty from 4.1.68 to 4.1.73
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
close https://github.com/apache/incubator-kyuubi/issues/1868
### _How was this patch tested?_
Pass CI
Closes #1869 from ulysses-you/netty.
Closes #1868
c197d0e3 [ulysses-you] Upgrate netty from 4.1.68 to 4.1.73
Authored-by: ulysses-you <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
dev/dependencyList | 16 ++++-
externals/kyuubi-flink-sql-engine/pom.xml | 8 ---
kyuubi-common/pom.xml | 5 --
.../org/apache/kyuubi/config/KyuubiConf.scala | 11 +++-
kyuubi-server/pom.xml | 5 ++
.../kyuubi/server/KyuubiMySQLFrontendService.scala | 2 +-
.../apache/kyuubi/server/mysql}/NettyUtils.scala | 19 ++----
pom.xml | 76 +++++++++++++++++++++-
8 files changed, 111 insertions(+), 31 deletions(-)
diff --git a/dev/dependencyList b/dev/dependencyList
index 8d51b96..3bb9e6e 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -79,7 +79,21 @@ metrics-core/4.2.8//metrics-core-4.2.8.jar
metrics-jmx/4.2.8//metrics-jmx-4.2.8.jar
metrics-json/4.2.8//metrics-json-4.2.8.jar
metrics-jvm/4.2.8//metrics-jvm-4.2.8.jar
-netty-all/4.1.68.Final//netty-all-4.1.68.Final.jar
+netty-all/4.1.73.Final//netty-all-4.1.73.Final.jar
+netty-buffer/4.1.73.Final//netty-buffer-4.1.73.Final.jar
+netty-codec/4.1.73.Final//netty-codec-4.1.73.Final.jar
+netty-common/4.1.73.Final//netty-common-4.1.73.Final.jar
+netty-handler/4.1.73.Final//netty-handler-4.1.73.Final.jar
+netty-resolver/4.1.73.Final//netty-resolver-4.1.73.Final.jar
+netty-tcnative-classes/2.0.46.Final//netty-tcnative-classes-2.0.46.Final.jar
+netty-transport-classes-epoll/4.1.73.Final//netty-transport-classes-epoll-4.1.73.Final.jar
+netty-transport-classes-kqueue/4.1.73.Final//netty-transport-classes-kqueue-4.1.73.Final.jar
+netty-transport-native-epoll/4.1.73.Final/linux-aarch_64/netty-transport-native-epoll-4.1.73.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.1.73.Final/linux-x86_64/netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.1.73.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.73.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.1.73.Final/osx-x86_64/netty-transport-native-kqueue-4.1.73.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.1.73.Final//netty-transport-native-unix-common-4.1.73.Final.jar
+netty-transport/4.1.73.Final//netty-transport-4.1.73.Final.jar
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8//paranamer-2.8.jar
scala-library/2.12.15//scala-library-2.12.15.jar
diff --git a/externals/kyuubi-flink-sql-engine/pom.xml
b/externals/kyuubi-flink-sql-engine/pom.xml
index fe5ee69..1012a0f 100644
--- a/externals/kyuubi-flink-sql-engine/pom.xml
+++ b/externals/kyuubi-flink-sql-engine/pom.xml
@@ -165,7 +165,6 @@
<include>com.google.guava:failureaccess</include>
<include>com.google.guava:guava</include>
<include>commons-codec:commons-codec</include>
- <include>io.netty:netty-all</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.apache.curator:curator-client</include>
<include>org.apache.curator:curator-framework</include>
@@ -206,13 +205,6 @@
<shadedPattern>${kyuubi.shade.packageName}.org.apache.commons.lang</shadedPattern>
</relocation>
<relocation>
- <pattern>io.netty</pattern>
-
<shadedPattern>${kyuubi.shade.packageName}.io.netty</shadedPattern>
- <includes>
- <include>io.netty.**</include>
- </includes>
- </relocation>
- <relocation>
<pattern>org.apache.hive.service.rpc.thrift</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>
<includes>
diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml
index 1b81bd9..e02987e 100644
--- a/kyuubi-common/pom.xml
+++ b/kyuubi-common/pom.xml
@@ -114,11 +114,6 @@
</dependency>
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<scope>test</scope>
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index ef986c5..8722b15 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -27,7 +27,6 @@ import scala.collection.JavaConverters._
import org.apache.kyuubi.{Logging, Utils}
import org.apache.kyuubi.engine.{EngineType, ShareLevel}
import org.apache.kyuubi.service.authentication.{AuthTypes, SaslQOP}
-import org.apache.kyuubi.util.NettyUtils.MAX_NETTY_THREADS
case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
import KyuubiConf._
@@ -516,6 +515,16 @@ object KyuubiConf {
.checkValue(p => p == 0 || (p > 1024 && p < 65535), "Invalid Port number")
.createWithDefault(3309)
+ /**
+ * Specifies an upper bound on the number of Netty threads that Kyuubi
requires by default.
+ * In practice, only 2-4 cores should be required to transfer roughly 10
Gb/s, and each core
+ * that we use will have an initial overhead of roughly 32 MB of off-heap
memory, which comes
+ * at a premium.
+ *
+ * Thus, this value should still retain maximum throughput and reduce wasted
off-heap memory
+ * allocation.
+ */
+ val MAX_NETTY_THREADS: Int = 8
val FRONTEND_MYSQL_NETTY_WORKER_THREADS: OptionalConfigEntry[Int] =
buildConf("frontend.mysql.netty.worker.threads")
.doc("Number of thread in the netty worker event loop of MySQL frontend
service. " +
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index ffbfb56..2665783 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -131,6 +131,11 @@
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-jaxrs2</artifactId>
</dependency>
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiMySQLFrontendService.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiMySQLFrontendService.scala
index c779e35..a8dc510 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiMySQLFrontendService.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiMySQLFrontendService.scala
@@ -30,10 +30,10 @@ import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.server.mysql._
+import org.apache.kyuubi.server.mysql.NettyUtils._
import org.apache.kyuubi.server.mysql.authentication.MySQLAuthHandler
import org.apache.kyuubi.service.{AbstractFrontendService, Serverable, Service}
import org.apache.kyuubi.util.NamedThreadFactory
-import org.apache.kyuubi.util.NettyUtils._
/**
* A frontend service implement MySQL protocol.
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NettyUtils.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/mysql/NettyUtils.scala
similarity index 77%
rename from kyuubi-common/src/main/scala/org/apache/kyuubi/util/NettyUtils.scala
rename to
kyuubi-server/src/main/scala/org/apache/kyuubi/server/mysql/NettyUtils.scala
index c5b6e2e..899af85 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NettyUtils.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/mysql/NettyUtils.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.kyuubi.util
+package org.apache.kyuubi.server.mysql
import java.util.concurrent.ThreadFactory
@@ -27,18 +27,9 @@ import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.nio.{NioServerSocketChannel, NioSocketChannel}
import io.netty.util.concurrent.DefaultThreadFactory
-object NettyUtils {
+import org.apache.kyuubi.config.KyuubiConf
- /**
- * Specifies an upper bound on the number of Netty threads that Kyuubi
requires by default.
- * In practice, only 2-4 cores should be required to transfer roughly 10
Gb/s, and each core
- * that we use will have an initial overhead of roughly 32 MB of off-heap
memory, which comes
- * at a premium.
- *
- * Thus, this value should still retain maximum throughput and reduce wasted
off-heap memory
- * allocation.
- */
- val MAX_NETTY_THREADS: Int = 8
+object NettyUtils {
val EPOLL_MODE: Boolean = Epoll.isAvailable
@@ -65,7 +56,7 @@ object NettyUtils {
* we will use Runtime get an approximate number of available cores.
*/
def defaultNumThreads(numUsableCores: Option[Int]): Int = numUsableCores
match {
- case Some(num) => min(num, MAX_NETTY_THREADS)
- case None => min(sys.runtime.availableProcessors, MAX_NETTY_THREADS)
+ case Some(num) => min(num, KyuubiConf.MAX_NETTY_THREADS)
+ case None => min(sys.runtime.availableProcessors,
KyuubiConf.MAX_NETTY_THREADS)
}
}
diff --git a/pom.xml b/pom.xml
index 41240c4..5278298 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
<kubernetes-client.version>5.12.1</kubernetes-client.version>
<ldapsdk.version>5.1.4</ldapsdk.version>
<mysql.jdbc.version>8.0.27</mysql.jdbc.version>
- <netty.version>4.1.68.Final</netty.version>
+ <netty.version>4.1.73.Final</netty.version>
<parquet.version>1.10.1</parquet.version>
<prometheus.version>0.14.1</prometheus.version>
<scalacheck.version>3.2.9.0</scalacheck.version>
@@ -745,6 +745,80 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-dns</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-haproxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-memcache</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-mqtt</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-redis</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-smtp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-socks</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-stomp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-xml</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver-dns</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+
<artifactId>netty-resolver-dns-classes-macos</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+
<artifactId>netty-resolver-dns-native-macos</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-rxtx</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-sctp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-udt</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
