This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 6fc7552ef [KYUUBI #6118] Migrate to shaded HMS client for getting
delegation token on server
6fc7552ef is described below
commit 6fc7552ef05c83cfb8a2cd56d80c02c6e8a033c7
Author: zhouyifan279 <[email protected]>
AuthorDate: Fri Mar 8 13:07:11 2024 +0800
[KYUUBI #6118] Migrate to shaded HMS client for getting delegation token on
server
# :mag: Description
## Issue References ๐
## Describe Your Solution ๐ง
Kyuubi Shaded 0.3 introduces a light
kyuubi-relocated-hive-metastore-client, for refreshing
delegation token, this PR aims to migrate from the vanilla HMS client to
this light shaded HMS client, then we can get rid of Hive dependencies,
especially the vulnerable thrift 0.9, from the Kyuubi server.
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
Pass GA.
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6118 from zhouyifan279/relocated-hms-client.
Closes #6118
4d7e50915 [Cheng Pan] comment
845e39f87 [Cheng Pan] notice
b4a58a614 [Cheng Pan] Update NOTICE
f4bfa9310 [zhouyifan279] Use kyuubi-relocated-hive-metastore-client
af17be1fe [zhouyifan279] Use kyuubi-relocated-hive-metastore-client
Lead-authored-by: zhouyifan279 <[email protected]>
Co-authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
LICENSE-binary | 10 ---------
NOTICE-binary | 24 ----------------------
dev/dependencyList | 10 ---------
kyuubi-server/pom.xml | 16 +++++++++++++++
...rg.apache.hadoop.security.token.TokenIdentifier | 2 +-
.../credentials/HiveDelegationTokenProvider.scala | 10 ++++-----
.../kyuubi/server/http/ThriftHttpServlet.scala | 24 +++++++++++++++++++---
.../HiveDelegationTokenProviderSuite.scala | 3 ++-
8 files changed, 45 insertions(+), 54 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 380c67d0c..4d3335913 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -230,14 +230,6 @@ com.google.guava:failureaccess
com.google.guava:guava
org.apache.hadoop:hadoop-client-api
org.apache.hadoop:hadoop-client-runtime
-org.apache.hive:hive-common
-org.apache.hive:hive-metastore
-org.apache.hive:hive-standalone-metastore
-org.apache.hive:hive-llap-client
-org.apache.hive:hive-serde
-org.apache.hive:hive-service-rpc
-org.apache.hive:hive-shims-0.23
-org.apache.hive:hive-shims-common
com.google.j2objc:j2objc-annotations
com.fasterxml.jackson.core:jackson-annotations
com.fasterxml.jackson.core:jackson-core
@@ -270,8 +262,6 @@ org.eclipse.jetty:jetty-servlet
org.eclipse.jetty:jetty-util-ajax
org.eclipse.jetty:jetty-util
org.eclipse.jetty:jetty-proxy
-org.apache.thrift:libfb303
-org.apache.thrift:libthrift
org.apache.logging.log4j:log4j-1.2-api
org.apache.logging.log4j:log4j-api
org.apache.logging.log4j:log4j-core
diff --git a/NOTICE-binary b/NOTICE-binary
index 747b61fff..b8104d39f 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -198,33 +198,12 @@ Copyright 2022 The Apache Software Foundation
Hive JDBC
Copyright 2022 The Apache Software Foundation
-Hive Llap Client
-Copyright 2022 The Apache Software Foundation
-
-Hive Metastore
-Copyright 2022 The Apache Software Foundation
-
-Hive Serde
-Copyright 2022 The Apache Software Foundation
-
Hive Service
Copyright 2022 The Apache Software Foundation
Hive Service RPC
Copyright 2022 The Apache Software Foundation
-Hive Shims 0.23
-Copyright 2022 The Apache Software Foundation
-
-Hive Shims Common
-Copyright 2022 The Apache Software Foundation
-
-Hive Standalone Metastore
-Copyright 2022 The Apache Software Foundation
-
-Hive Storage API
-Copyright 2020 The Apache Software Foundation
-
Apache HttpClient
Copyright 1999-2020 The Apache Software Foundation
@@ -798,9 +777,6 @@ Permission to use, copy, modify and distribute UnixCrypt
for non-commercial or commercial purposes and without fee is
granted provided that the copyright notice appears in all copies.
-Apache Thrift
-Copyright 2006-2010 The Apache Software Foundation.
-
Apache Log4j 1.x Compatibility API
Copyright 1999-2022 The Apache Software Foundation
diff --git a/dev/dependencyList b/dev/dependencyList
index 4143b511a..394568cf3 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -51,14 +51,6 @@ gson/2.10.1//gson-2.10.1.jar
guava/32.0.1-jre//guava-32.0.1-jre.jar
hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
-hive-common/3.1.3//hive-common-3.1.3.jar
-hive-metastore/3.1.3//hive-metastore-3.1.3.jar
-hive-serde/3.1.3//hive-serde-3.1.3.jar
-hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar
-hive-shims-0.23/3.1.3//hive-shims-0.23-3.1.3.jar
-hive-shims-common/3.1.3//hive-shims-common-3.1.3.jar
-hive-standalone-metastore/3.1.3//hive-standalone-metastore-3.1.3.jar
-hive-storage-api/2.7.0//hive-storage-api-2.7.0.jar
hk2-api/2.6.1//hk2-api-2.6.1.jar
hk2-locator/2.6.1//hk2-locator-2.6.1.jar
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -133,8 +125,6 @@ kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar
kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar
kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar
kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar
-libfb303/0.9.3//libfb303-0.9.3.jar
-libthrift/0.9.3//libthrift-0.9.3.jar
log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar
log4j-api/2.20.0//log4j-api-2.20.0.jar
log4j-core/2.20.0//log4j-core-2.20.0.jar
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index f187ebff1..cb9c39a11 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -78,6 +78,12 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-relocated-hive-metastore-client</artifactId>
+ <version>${kyuubi-relocated.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
@@ -107,6 +113,7 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -119,6 +126,7 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-standalone-metastore</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -131,6 +139,7 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-serde</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -143,6 +152,7 @@
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-common</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -155,6 +165,7 @@
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-0.23</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -167,6 +178,7 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -179,6 +191,7 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${hive.storage-api.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -190,16 +203,19 @@
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libfb303</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-service-rpc</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
diff --git
a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 65e2965c0..f1f00d9aa 100644
---
a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++
b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
+org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
index 128346de5..f1966f13d 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
@@ -18,9 +18,6 @@
package org.apache.kyuubi.credentials
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.metastore.{IMetaStoreClient,
RetryingMetaStoreClient}
-import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, SecurityUtil}
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
@@ -28,6 +25,9 @@ import org.apache.hadoop.security.token.Token
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.shaded.hive.metastore.{IMetaStoreClient,
RetryingMetaStoreClient}
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf
+import
org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with
Logging {
@@ -38,7 +38,7 @@ class HiveDelegationTokenProvider extends
HadoopDelegationTokenProvider with Log
override def serviceName: String = "hive"
override def initialize(hadoopConf: Configuration, kyuubiConf: KyuubiConf):
Unit = {
- val conf = new HiveConf(hadoopConf, classOf[HiveConf])
+ val conf = MetastoreConf.newMetastoreConf(hadoopConf)
val metastoreUris = conf.getTrimmed("hive.metastore.uris", "")
// SQL engine requires token alias to be `hive.metastore.uris`
tokenAlias = new Text(metastoreUris)
@@ -51,7 +51,7 @@ class HiveDelegationTokenProvider extends
HadoopDelegationTokenProvider with Log
principal = conf.getTrimmed(principalKey, "")
require(principal.nonEmpty, s"Hive principal $principalKey undefined")
- client = Some(RetryingMetaStoreClient.getProxy(conf, false))
+ client = Some(RetryingMetaStoreClient.getProxy(conf))
info(s"Created HiveMetaStoreClient with metastore uris $metastoreUris")
}
}
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
index 980f35d70..d781faf3b 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
@@ -26,8 +26,6 @@ import javax.ws.rs.core.NewCookie
import scala.collection.mutable
-import org.apache.hadoop.hive.shims.Utils
-
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
@@ -57,6 +55,8 @@ class ThriftHttpServlet(
private var isHttpOnlyCookie = false
private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For"
private val authenticationFilter = new AuthenticationFilter(conf)
+ private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER"
+ private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD",
"TRACE")
override def init(): Unit = {
isCookieAuthEnabled =
conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED)
@@ -82,7 +82,7 @@ class ThriftHttpServlet(
var requireNewCookie: Boolean = false
try {
if (conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_XSRF_FILTER_ENABLED)) {
- val continueProcessing = Utils.doXsrfFilter(request, response, null,
null)
+ val continueProcessing = doXsrfFilter(request, response)
if (!continueProcessing) {
warn("Request did not have valid XSRF header, rejecting.")
return
@@ -303,4 +303,22 @@ class ThriftHttpServlet(
null
}
+
+ private def doXsrfFilter(
+ httpRequest: HttpServletRequest,
+ response: HttpServletResponse): Boolean = {
+ if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(httpRequest.getMethod)
+ || httpRequest.getHeader(XSRF_HEADER_DEFAULT) != null) {
+ true
+ } else {
+ response.sendError(
+ HttpServletResponse.SC_BAD_REQUEST,
+ "Missing Required Header for Vulnerability Protection")
+ // scalastyle:off println
+ response.getWriter.println(
+ "XSRF filter denial, requests must contain header : " +
XSRF_HEADER_DEFAULT)
+ // scalastyle:on println
+ false
+ }
+ }
}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
index 6c0370f55..76d3e5614 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.conf.HiveConf.ConfVars._
import org.apache.hadoop.hive.metastore.{HiveMetaException, HiveMetaStore}
-import org.apache.hadoop.hive.metastore.security.{DelegationTokenIdentifier,
HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
+import org.apache.hadoop.hive.metastore.security.{HadoopThriftAuthBridge,
HadoopThriftAuthBridge23}
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.authorize.ProxyUsers
@@ -44,6 +44,7 @@ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
import org.apache.kyuubi.{KerberizedTestHelper, Logging, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.credentials.LocalMetaServer.defaultHiveConf
+import
org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {
