Repository: hive Updated Branches: refs/heads/llap 86e1c89ea -> b87f63cad
HIVE-11720: Allow HiveServer2 to set custom http request/response header size (Vaibhav Gumashta reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7003aa46 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7003aa46 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7003aa46 Branch: refs/heads/llap Commit: 7003aa46d45980030e94f421cd0beca273511440 Parents: 0ca9ff8 Author: Vaibhav Gumashta <[email protected]> Authored: Mon Oct 5 11:57:56 2015 -0700 Committer: Vaibhav Gumashta <[email protected]> Committed: Mon Oct 5 11:57:56 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 4 ++ .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 10 ++-- .../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 55 ++++++++++++++++++++ .../cli/thrift/ThriftHttpCLIService.java | 7 +++ 4 files changed, 71 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7003aa46/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7f632bc..54a529e 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1900,6 +1900,10 @@ public class HiveConf extends Configuration { new TimeValidator(TimeUnit.SECONDS), "Keepalive time for an idle http worker thread. When the number of workers exceeds min workers, " + "excessive threads are killed after this time interval."), + HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE("hive.server2.thrift.http.request.header.size", 6*1024, + "Request header size in bytes, when using HTTP transport mode. Jetty defaults used."), + HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE("hive.server2.thrift.http.response.header.size", 6*1024, + "Response header size in bytes, when using HTTP transport mode. Jetty defaults used."), // Cookie based authentication when using HTTP Transport HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED("hive.server2.thrift.http.cookie.auth.enabled", true, http://git-wip-us.apache.org/repos/asf/hive/blob/7003aa46/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index adb8a71..9f051ed 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -336,16 +336,16 @@ public class MiniHS2 extends AbstractHiveService { hiveConfExt = (hiveConfExt == null ? "" : hiveConfExt); String krbConfig = ""; if (isUseMiniKdc()) { - krbConfig = ";principal=" + serverPrincipal; + krbConfig = "principal=" + serverPrincipal; } if (isHttpTransportMode()) { - hiveConfExt = "hive.server2.transport.mode=http;hive.server2.thrift.http.path=cliservice;" - + hiveConfExt; + sessionConfExt = "transportMode=http;httpPath=cliservice;" + sessionConfExt; } + String baseJdbcURL = getBaseJdbcURL() + dbName + ";" + krbConfig + ";" + sessionConfExt; if (!hiveConfExt.trim().equals("")) { - hiveConfExt = "?" + hiveConfExt; + baseJdbcURL = "?" + hiveConfExt; } - return getBaseJdbcURL() + dbName + krbConfig + sessionConfExt + hiveConfExt; + return baseJdbcURL; } /** http://git-wip-us.apache.org/repos/asf/hive/blob/7003aa46/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 306e3fe..8ba2a12 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -45,6 +45,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -668,4 +669,58 @@ public class TestJdbcWithMiniHS2 { fs.getFileStatus(scratchDirPath).getPermission()); } } + + /** + * Test for http header size + * @throws Exception + */ + @Test + public void testHttpHeaderSize() throws Exception { + // Stop HiveServer2 + if (miniHS2.isStarted()) { + miniHS2.stop(); + } + HiveConf conf = new HiveConf(); + conf.set("hive.server2.transport.mode", "http"); + conf.setInt("hive.server2.thrift.http.request.header.size", 1024); + conf.setInt("hive.server2.thrift.http.response.header.size", 1024); + miniHS2 = new MiniHS2(conf); + Map<String, String> confOverlay = new HashMap<String, String>(); + miniHS2.start(confOverlay); + + // Username is added to the request header + String userName = StringUtils.leftPad("*", 100); + // This should go fine, since header should be less than the configured header size + try { + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password"); + } catch (Exception e) { + fail("Not expecting exception: " + e); + } + + // This should fail with given HTTP response code 413 in error message, since header is more + // than the configured the header size + userName = StringUtils.leftPad("*", 2000); + try { + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password"); + } catch (Exception e) { + assertTrue("Header exception thrown", e != null); + assertTrue(e.getMessage().contains("HTTP Response code: 413")); + } + + // Stop HiveServer2 to increase header size + if (miniHS2.isStarted()) { + miniHS2.stop(); + } + conf.setInt("hive.server2.thrift.http.request.header.size", 3000); + conf.setInt("hive.server2.thrift.http.response.header.size", 3000); + miniHS2 = new MiniHS2(conf); + miniHS2.start(confOverlay); + + // This should now go fine, since we increased the configured header size + try { + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password"); + } catch (Exception e) { + fail("Not expecting exception: " + e); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/7003aa46/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index 046958e..a940bd6 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -75,6 +75,13 @@ public class ThriftHttpCLIService extends ThriftCLIService { // Connector configs SelectChannelConnector connector = new SelectChannelConnector(); + // Configure header size + int requestHeaderSize = + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_REQUEST_HEADER_SIZE); + int responseHeaderSize = + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_RESPONSE_HEADER_SIZE); + connector.setRequestHeaderSize(requestHeaderSize); + connector.setResponseHeaderSize(responseHeaderSize); boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL); String schemeName = useSsl ? "https" : "http"; // Change connector if SSL is used
