This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1afd305ba60 HIVE-26670: Track every single HTTP request between
beeline and hs2 (#3710) (Laszlo Bodor reviewed by Ayush Saxena, Chris Nauroth)
1afd305ba60 is described below
commit 1afd305ba602bdeac0e5608ce858ef5c2dab6e43
Author: Bodor Laszlo <[email protected]>
AuthorDate: Tue Nov 1 13:47:21 2022 +0100
HIVE-26670: Track every single HTTP request between beeline and hs2 (#3710)
(Laszlo Bodor reviewed by Ayush Saxena, Chris Nauroth)
---
.../src/main/resources/beeline-log4j2.properties | 6 ++-
.../org/apache/hadoop/hive/conf/Constants.java | 2 +
.../java/org/apache/hive/jdbc/HiveConnection.java | 33 ++++++++++++--
.../hive/jdbc/HttpDefaultResponseInterceptor.java | 42 ++++++++++++++++++
.../hive/jdbc/HttpRequestInterceptorBase.java | 35 ++++++++++++++-
.../hive/jdbc/HttpResponseInterceptorBase.java | 26 +++++++++++
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 2 +
.../org/apache/hive/jdbc/TestHiveConnection.java | 21 ++++++++-
.../hive/jdbc/TestHttpRequestInterceptor.java | 51 ++++++++++++++++++++++
.../hive/service/cli/thrift/ThriftHttpServlet.java | 11 ++++-
10 files changed, 220 insertions(+), 9 deletions(-)
diff --git a/beeline/src/main/resources/beeline-log4j2.properties
b/beeline/src/main/resources/beeline-log4j2.properties
index 103d72253f4..1300609ac87 100644
--- a/beeline/src/main/resources/beeline-log4j2.properties
+++ b/beeline/src/main/resources/beeline-log4j2.properties
@@ -33,12 +33,16 @@ appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
# list of all loggers
-loggers = HiveConnection
+loggers = HiveConnection, HiveJDBC
# HiveConnection logs useful info for dynamic service discovery
logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection
logger.HiveConnection.level = INFO
+# all jdbc class logs can be useful while investigating connection issues,
connection tracking
+logger.HiveJDBC.name = org.apache.hive.jdbc
+logger.HiveJDBC.level = INFO
+
# root logger
rootLogger.level = ${sys:hive.log.level}
rootLogger.appenderRefs = root
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
index decdc648a6c..c923c2bcce3 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
@@ -103,4 +103,6 @@ public class Constants {
public static final Pattern COMPACTION_POOLS_PATTERN =
Pattern.compile("hive\\.compactor\\.worker\\.(.*)\\.threads");
public static final String HIVE_COMPACTOR_WORKER_POOL =
"hive.compactor.worker.pool";
+ public static final String HTTP_HEADER_REQUEST_TRACK = "Request-Track";
+ public static final String TIME_POSTFIX_REQUEST_TRACK = "_TIME";
}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 4a6fb7c423d..c91416a02b7 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -65,7 +65,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReentrantLock;
@@ -120,7 +119,6 @@ import
org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp;
import org.apache.hive.service.rpc.thrift.TSessionHandle;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
-import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.NoHttpResponseException;
import org.apache.http.HttpStatus;
@@ -144,6 +142,7 @@ import
org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.Args;
+import org.apache.thrift.TBaseHelper;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.THttpClient;
@@ -151,6 +150,7 @@ import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.function.Supplier;
/**
* HiveConnection.
@@ -559,7 +559,7 @@ public class HiveConnection implements java.sql.Connection {
CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null;
HttpClientBuilder httpClientBuilder = null;
// Request interceptor for any request pre-processing logic
- HttpRequestInterceptor requestInterceptor;
+ HttpRequestInterceptorBase requestInterceptor;
Map<String, String> additionalHttpHeaders = new HashMap<String, String>();
Map<String, String> customCookies = new HashMap<String, String>();
@@ -752,8 +752,12 @@ public class HiveConnection implements java.sql.Connection
{
httpClientBuilder
.setRedirectStrategy(new
HiveJdbcSamlRedirectStrategy(browserClient));
}
+
+ requestInterceptor.setRequestTrackingEnabled(isRequestTrackingEnabled());
+
// Add the request interceptor to the client builder
- httpClientBuilder.addInterceptorFirst(requestInterceptor);
+
httpClientBuilder.addInterceptorFirst(requestInterceptor.sessionId(getSessionId()));
+ httpClientBuilder.addInterceptorLast(new HttpDefaultResponseInterceptor());
// Add an interceptor to add in an XSRF header
httpClientBuilder.addInterceptorLast(new XsrfHttpRequestInterceptor());
@@ -813,6 +817,27 @@ public class HiveConnection implements java.sql.Connection
{
return httpClientBuilder.build();
}
+ private boolean isRequestTrackingEnabled() {
+ return
Boolean.parseBoolean(sessConfMap.get(JdbcConnectionParams.JDBC_PARAM_REQUEST_TRACK));
+ }
+
+ /**
+ * Creates a sessionId Supplier for interceptors. When interceptors are
instantiated,
+ * there is no session yet (sessHandle is null) so this Supplier can take
care
+ * of the sessionId in a lazy way.
+ */
+ private Supplier<String> getSessionId() {
+ Supplier<String> sessionId = () -> {
+ if (sessHandle == null) {
+ return "NO_SESSION";
+ }
+ StringBuilder b = new StringBuilder();
+ TBaseHelper.toString(sessHandle.getSessionId().bufferForGuid(), b);
+ return b.toString().replaceAll("\\s", "");
+ };
+ return sessionId;
+ }
+
private String getJWT() {
String jwtCredential = getJWTStringFromSession();
if (jwtCredential == null || jwtCredential.isEmpty()) {
diff --git
a/jdbc/src/java/org/apache/hive/jdbc/HttpDefaultResponseInterceptor.java
b/jdbc/src/java/org/apache/hive/jdbc/HttpDefaultResponseInterceptor.java
new file mode 100644
index 00000000000..94e5b9125d0
--- /dev/null
+++ b/jdbc/src/java/org/apache/hive/jdbc/HttpDefaultResponseInterceptor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.jdbc;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.conf.Constants;
+import org.apache.hadoop.util.Time;
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.protocol.HttpContext;
+
+public class HttpDefaultResponseInterceptor extends
HttpResponseInterceptorBase {
+
+ @Override
+ public void process(HttpResponse response, HttpContext context) throws
HttpException, IOException {
+ String trackHeader = (String)
context.getAttribute(Constants.HTTP_HEADER_REQUEST_TRACK);
+ if (trackHeader == null) {
+ return;
+ }
+ String trackTimeHeader = trackHeader +
Constants.TIME_POSTFIX_REQUEST_TRACK;
+ long elapsed = Time.monotonicNow() - (long)
context.getAttribute(trackTimeHeader);
+ LOG.info("Response to {} in {} ms", trackHeader, elapsed);
+ context.removeAttribute(Constants.HTTP_HEADER_REQUEST_TRACK);
+ context.removeAttribute(trackTimeHeader);
+ }
+}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
b/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
index eeaa48a5d11..862d299643e 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
@@ -19,21 +19,33 @@
package org.apache.hive.jdbc;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+import org.apache.hadoop.hive.conf.Constants;
+import org.apache.hadoop.util.Time;
import org.apache.http.Header;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.client.CookieStore;
import org.apache.http.protocol.HttpContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class HttpRequestInterceptorBase implements
HttpRequestInterceptor {
+ protected final Logger LOG = LoggerFactory.getLogger(getClass());
+
CookieStore cookieStore;
boolean isCookieEnabled;
String cookieName;
boolean isSSL;
Map<String, String> additionalHeaders;
Map<String, String> customCookies;
+ private Supplier<String> sessionId = null;
+ private boolean requestTrackingEnabled;
+ private final AtomicLong requestTrackCounter = new AtomicLong();
// Abstract function to add HttpAuth Header
protected abstract void addHttpAuthHeader(HttpRequest httpRequest,
HttpContext httpContext)
@@ -45,7 +57,7 @@ public abstract class HttpRequestInterceptorBase implements
HttpRequestIntercept
this.isCookieEnabled = (cs != null);
this.cookieName = cn;
this.isSSL = isSSL;
- this.additionalHeaders = additionalHeaders;
+ this.additionalHeaders = additionalHeaders == null ? new HashMap<>() :
additionalHeaders;
this.customCookies = customCookies;
}
@@ -77,6 +89,14 @@ public abstract class HttpRequestInterceptorBase implements
HttpRequestIntercept
if (isCookieEnabled) {
httpContext.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY,
Utils.HIVE_SERVER2_CONST_FALSE);
}
+
+ if (requestTrackingEnabled) {
+ String trackHeader = getNewTrackHeader();
+ LOG.info("{}:{}", Constants.HTTP_HEADER_REQUEST_TRACK, trackHeader);
+ additionalHeaders.put(Constants.HTTP_HEADER_REQUEST_TRACK,
trackHeader);
+ httpContext.setAttribute(Constants.HTTP_HEADER_REQUEST_TRACK,
trackHeader);
+ httpContext.setAttribute(trackHeader +
Constants.TIME_POSTFIX_REQUEST_TRACK, Time.monotonicNow());
+ }
// Insert the additional http headers
if (additionalHeaders != null) {
for (Map.Entry<String, String> entry : additionalHeaders.entrySet()) {
@@ -102,4 +122,17 @@ public abstract class HttpRequestInterceptorBase
implements HttpRequestIntercept
throw new HttpException(e.getMessage(), e);
}
}
+
+ protected String getNewTrackHeader() {
+ return String.format("HIVE_%s_%020d", sessionId.get(),
requestTrackCounter.incrementAndGet());
+ }
+
+ public HttpRequestInterceptor sessionId(Supplier<String> sessionId) {
+ this.sessionId = sessionId;
+ return this;
+ }
+
+ public void setRequestTrackingEnabled(boolean requestTrackingEnabled) {
+ this.requestTrackingEnabled = requestTrackingEnabled;
+ }
}
diff --git
a/jdbc/src/java/org/apache/hive/jdbc/HttpResponseInterceptorBase.java
b/jdbc/src/java/org/apache/hive/jdbc/HttpResponseInterceptorBase.java
new file mode 100644
index 00000000000..6036a12d38f
--- /dev/null
+++ b/jdbc/src/java/org/apache/hive/jdbc/HttpResponseInterceptorBase.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.jdbc;
+
+import org.apache.http.HttpResponseInterceptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class HttpResponseInterceptorBase implements
HttpResponseInterceptor {
+ protected final Logger LOG = LoggerFactory.getLogger(getClass());
+}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index c1be6a52df4..a855d4e2a5d 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -156,6 +156,8 @@ public class Utils {
static final String DEFAULT_COOKIE_NAMES_HS2 = "hive.server2.auth";
// The http header prefix for additional headers which have to be appended
to the request
static final String HTTP_HEADER_PREFIX = "http.header.";
+ // Request tracking
+ static final String JDBC_PARAM_REQUEST_TRACK = "requestTrack";
// Set the fetchSize
static final String FETCH_SIZE = "fetchSize";
static final String INIT_FILE = "initFile";
diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHiveConnection.java
b/jdbc/src/test/org/apache/hive/jdbc/TestHiveConnection.java
index bcd2608e1ba..71abe784488 100644
--- a/jdbc/src/test/org/apache/hive/jdbc/TestHiveConnection.java
+++ b/jdbc/src/test/org/apache/hive/jdbc/TestHiveConnection.java
@@ -18,15 +18,17 @@
package org.apache.hive.jdbc;
+import java.io.IOException;
+import java.sql.SQLException;
+
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
+import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-
public class TestHiveConnection {
private static final String EXISTING_TOKEN = "ExistingToken";
@@ -57,4 +59,19 @@ public class TestHiveConnection {
String tokenStr = fetcher.getTokenFromCredential(creds, EXISTING_TOKEN);
Assert.assertEquals("Token string form is not as expected.",
EXPECTED_TOKEN_STRING_FORM, tokenStr);
}
+
+ @Test
+ public void testHiveConnectionParameters() throws SQLException,
ZooKeeperHiveClientException {
+ JdbcConnectionParams params = Utils.parseURL(
+
"jdbc:hive2://hello.host:10002/default;transportMode=http;httpPath=cliservice;socketTimeout=60;requestTrack=true;");
+
+ Assert.assertEquals("hello.host", params.getHost());
+ Assert.assertEquals("default", params.getDbName());
+ Assert.assertEquals(10002, params.getPort());
+
+ Assert.assertEquals("http",
params.getSessionVars().get(JdbcConnectionParams.TRANSPORT_MODE));
+ Assert.assertEquals("cliservice",
params.getSessionVars().get(JdbcConnectionParams.HTTP_PATH));
+ Assert.assertEquals("60",
params.getSessionVars().get(JdbcConnectionParams.SOCKET_TIMEOUT));
+ Assert.assertEquals("true",
params.getSessionVars().get(JdbcConnectionParams.JDBC_PARAM_REQUEST_TRACK));
+ }
}
diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHttpRequestInterceptor.java
b/jdbc/src/test/org/apache/hive/jdbc/TestHttpRequestInterceptor.java
new file mode 100644
index 00000000000..4c56aa6e7b1
--- /dev/null
+++ b/jdbc/src/test/org/apache/hive/jdbc/TestHttpRequestInterceptor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.hadoop.hive.conf.Constants;
+import org.apache.http.HttpException;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.message.BasicHttpRequest;
+import org.apache.http.protocol.BasicHttpContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHttpRequestInterceptor {
+
+ @Test
+ public void testRequestTrackHeader() throws HttpException, IOException {
+ HttpRequestInterceptorBase requestInterceptor = getInterceptor();
+ requestInterceptor.setRequestTrackingEnabled(true);
+ requestInterceptor.sessionId(() -> "sessionId");
+ requestInterceptor.process(new BasicHttpRequest("POST", "uri"), new
BasicHttpContext());
+
+
Assert.assertTrue(requestInterceptor.additionalHeaders.containsKey(Constants.HTTP_HEADER_REQUEST_TRACK));
+ Assert.assertEquals("HIVE_sessionId_00000000000000000001",
+
requestInterceptor.additionalHeaders.get(Constants.HTTP_HEADER_REQUEST_TRACK));
+ }
+
+ private HttpRequestInterceptorBase getInterceptor() {
+ HttpRequestInterceptorBase requestInterceptor = new
HttpBasicAuthInterceptor("user", "pass", new BasicCookieStore(),
+ "cookieName", false, new HashMap<>(), new HashMap<>());
+ return requestInterceptor;
+ }
+}
diff --git
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index bbb74e0c7ca..0c192f45ca4 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -24,7 +24,6 @@ import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.security.SecureRandom;
-import java.text.ParseException;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
@@ -45,6 +44,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.io.ByteStreams;
+import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
@@ -153,6 +153,8 @@ public class ThriftHttpServlet extends TServlet {
String clientIpAddress;
boolean requireNewCookie = false;
+ logTrackingHeaderIfAny(request);
+
try {
if
(hiveConf.getBoolean(ConfVars.HIVE_SERVER2_XSRF_FILTER_ENABLED.varname,false)){
boolean continueProcessing =
Utils.doXsrfFilter(request,response,null,null);
@@ -310,6 +312,13 @@ public class ThriftHttpServlet extends TServlet {
}
}
+ private void logTrackingHeaderIfAny(HttpServletRequest request) {
+ if (request.getHeader(Constants.HTTP_HEADER_REQUEST_TRACK) != null) {
+ String requestTrackHeader =
request.getHeader(Constants.HTTP_HEADER_REQUEST_TRACK);
+ LOG.info("{}:{}", Constants.HTTP_HEADER_REQUEST_TRACK,
requestTrackHeader);
+ }
+ }
+
private String validateJWT(HttpServletRequest request, HttpServletResponse
response)
throws HttpAuthenticationException {
Preconditions.checkState(jwtValidator != null, "JWT validator should have
been set");
