Repository: hive
Updated Branches:
  refs/heads/master 9cfdd4bb0 -> 47d904675


HIVE-14876 : make the number of rows to fetch from various HS2 clients/servers 
configurable (Sergey Shelukhin, reviewed by Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47d90467
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47d90467
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47d90467

Branch: refs/heads/master
Commit: 47d904675d04539bdef96ed26e00b1786251e9f2
Parents: 9cfdd4b
Author: Sergey Shelukhin <ser...@apache.org>
Authored: Wed Oct 12 16:01:29 2016 -0700
Committer: Sergey Shelukhin <ser...@apache.org>
Committed: Wed Oct 12 16:01:29 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/conf/HiveConf.java   |  8 +++++---
 .../java/org/apache/hive/service/cli/CLIService.java |  4 +++-
 .../apache/hive/service/cli/CLIServiceClient.java    | 13 ++++++++++---
 .../hive/service/cli/EmbeddedCLIServiceClient.java   |  5 ++++-
 .../apache/hive/service/cli/operation/Operation.java |  6 +-----
 .../hive/service/cli/operation/OperationManager.java |  8 +-------
 .../cli/thrift/RetryingThriftCLIServiceClient.java   |  8 +++++---
 .../service/cli/thrift/ThriftCLIServiceClient.java   | 15 ++++++++++++---
 .../cli/TestRetryingThriftCLIServiceClient.java      |  2 +-
 9 files changed, 42 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index e6ca361..18b98e9 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2510,9 +2510,11 @@ public class HiveConf extends Configuration {
       "Whether we should serialize the Thrift structures used in JDBC 
ResultSet RPC in task nodes.\n " +
       "We use SequenceFile and ThriftJDBCBinarySerDe to read and write the 
final results if this is true."),
     // TODO: Make use of this config to configure fetch size
-    
HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE("hive.server2.thrift.resultset.max.fetch.size",
 1000,
-      "Max number of rows sent in one Fetch RPC call by the server to the 
client."),
-
+    
HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE("hive.server2.thrift.resultset.max.fetch.size",
+        10000, "Max number of rows sent in one Fetch RPC call by the server to 
the client."),
+    
HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE("hive.server2.resultset.default.fetch.size",
 10000,
+        "The number of rows sent in one Fetch RPC call by the server to the 
client, if not\n" +
+        "specified by the client."),
     HIVE_SERVER2_XSRF_FILTER_ENABLED("hive.server2.xsrf.filter.enabled",false,
         "If enabled, HiveServer2 will block any requests made to it over http 
" +
         "if an X-XSRF-HEADER header is not present"),

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/CLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java 
b/service/src/java/org/apache/hive/service/cli/CLIService.java
index 662e55c..6d07d0c 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIService.java
@@ -68,6 +68,7 @@ public class CLIService extends CompositeService implements 
ICLIService {
   private UserGroupInformation httpUGI;
   // The HiveServer2 instance running this service
   private final HiveServer2 hiveServer2;
+  private int defaultFetchRows;
 
   public CLIService(HiveServer2 hiveServer2) {
     super(CLIService.class.getSimpleName());
@@ -78,6 +79,7 @@ public class CLIService extends CompositeService implements 
ICLIService {
   public synchronized void init(HiveConf hiveConf) {
     this.hiveConf = hiveConf;
     sessionManager = new SessionManager(hiveServer2);
+    defaultFetchRows = 
hiveConf.getIntVar(ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE);
     addService(sessionManager);
     //  If the hadoop cluster is secure, do a kerberos login for the service 
from the keytab
     if (UserGroupInformation.isSecurityEnabled()) {
@@ -500,7 +502,7 @@ public class CLIService extends CompositeService implements 
ICLIService {
   public RowSet fetchResults(OperationHandle opHandle)
       throws HiveSQLException {
     return fetchResults(opHandle, Operation.DEFAULT_FETCH_ORIENTATION,
-        Operation.DEFAULT_FETCH_MAX_ROWS, FetchType.QUERY_OUTPUT);
+        defaultFetchRows, FetchType.QUERY_OUTPUT);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java 
b/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java
index 3155c23..4b84872 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java
@@ -20,6 +20,9 @@ package org.apache.hive.service.cli;
 
 import java.util.Collections;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hive.service.auth.HiveAuthFactory;
 
 
@@ -28,7 +31,11 @@ import org.apache.hive.service.auth.HiveAuthFactory;
  *
  */
 public abstract class CLIServiceClient implements ICLIService {
-  private static final long DEFAULT_MAX_ROWS = 1000;
+  protected int defaultFetchRows = 
ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal;
+
+  public CLIServiceClient(Configuration conf) {
+    defaultFetchRows = HiveConf.getIntVar(conf, 
ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE);
+  }
 
   public SessionHandle openSession(String username, String password)
       throws HiveSQLException {
@@ -37,8 +44,8 @@ public abstract class CLIServiceClient implements ICLIService 
{
 
   @Override
   public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException 
{
-    // TODO: provide STATIC default value
-    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT, 
DEFAULT_MAX_ROWS, FetchType.QUERY_OUTPUT);
+    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT,
+        defaultFetchRows, FetchType.QUERY_OUTPUT);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
----------------------------------------------------------------------
diff --git 
a/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java 
b/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
index 86e9bb1..2a3bcca 100644
--- a/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
+++ b/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java
@@ -21,6 +21,7 @@ package org.apache.hive.service.cli;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hive.service.auth.HiveAuthFactory;
 
 
@@ -31,7 +32,9 @@ import org.apache.hive.service.auth.HiveAuthFactory;
 public class EmbeddedCLIServiceClient extends CLIServiceClient {
   private final ICLIService cliService;
 
-  public EmbeddedCLIServiceClient(ICLIService cliService) {
+  // TODO: this doesn't appear to be used anywhere.
+  public EmbeddedCLIServiceClient(ICLIService cliService, Configuration conf) {
+    super(conf);
     this.cliService = cliService;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/operation/Operation.java
----------------------------------------------------------------------
diff --git 
a/service/src/java/org/apache/hive/service/cli/operation/Operation.java 
b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
index 36c6f93..76a1286 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java
@@ -64,7 +64,7 @@ public abstract class Operation {
   private final OperationHandle opHandle;
   public static final FetchOrientation DEFAULT_FETCH_ORIENTATION = 
FetchOrientation.FETCH_NEXT;
   public static final Logger LOG = 
LoggerFactory.getLogger(Operation.class.getName());
-  public static final long DEFAULT_FETCH_MAX_ROWS = 100;
+  public static final long DEFAULT_FETCH_MAX_ROWS = 100; // TODO# remove this
   protected boolean hasResultSet;
   protected volatile HiveSQLException operationException;
   protected volatile Future<?> backgroundHandle;
@@ -362,10 +362,6 @@ public abstract class Operation {
 
   public abstract RowSet getNextRowSet(FetchOrientation orientation, long 
maxRows) throws HiveSQLException;
 
-  public RowSet getNextRowSet() throws HiveSQLException {
-    return getNextRowSet(FetchOrientation.FETCH_NEXT, DEFAULT_FETCH_MAX_ROWS);
-  }
-
   public String getTaskStatus() throws HiveSQLException {
     return null;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git 
a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java 
b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index dab8ba1..3ae51b3 100644
--- 
a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ 
b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -299,14 +299,8 @@ public class OperationManager extends AbstractService {
     return getOperation(opHandle).getResultSetSchema();
   }
 
-  public RowSet getOperationNextRowSet(OperationHandle opHandle)
-      throws HiveSQLException {
-    return getOperation(opHandle).getNextRowSet();
-  }
-
   public RowSet getOperationNextRowSet(OperationHandle opHandle,
-      FetchOrientation orientation, long maxRows)
-          throws HiveSQLException {
+      FetchOrientation orientation, long maxRows) throws HiveSQLException {
     return getOperation(opHandle).getNextRowSet(orientation, maxRows);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
----------------------------------------------------------------------
diff --git 
a/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
 
b/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
index 933750b..85b82b6 100644
--- 
a/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
+++ 
b/service/src/java/org/apache/hive/service/cli/thrift/RetryingThriftCLIServiceClient.java
@@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
 import javax.security.sasl.SaslException;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.auth.PlainSaslHelper;
 import org.apache.hive.service.cli.CLIServiceClient;
@@ -73,7 +74,8 @@ public class RetryingThriftCLIServiceClient implements 
InvocationHandler {
     private final ICLIService cliService;
     private TTransport tTransport;
 
-    public CLIServiceClientWrapper(ICLIService icliService, TTransport 
tTransport) {
+    public CLIServiceClientWrapper(ICLIService icliService, TTransport 
tTransport, HiveConf conf) {
+      super(conf);
       cliService = icliService;
       this.tTransport = tTransport;
     }
@@ -249,7 +251,7 @@ public class RetryingThriftCLIServiceClient implements 
InvocationHandler {
     ICLIService cliService =
       (ICLIService) 
Proxy.newProxyInstance(RetryingThriftCLIServiceClient.class.getClassLoader(),
         CLIServiceClient.class.getInterfaces(), retryClient);
-    return new CLIServiceClientWrapper(cliService, tTransport);
+    return new CLIServiceClientWrapper(cliService, tTransport, conf);
   }
 
   protected TTransport connectWithRetry(int retries) throws HiveSQLException {
@@ -299,7 +301,7 @@ public class RetryingThriftCLIServiceClient implements 
InvocationHandler {
 
     TProtocol protocol = new TBinaryProtocol(transport);
     transport.open();
-    base = new ThriftCLIServiceClient(new TCLIService.Client(protocol));
+    base = new ThriftCLIServiceClient(new TCLIService.Client(protocol), conf);
     LOG.info("Connected!");
     return transport;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
----------------------------------------------------------------------
diff --git 
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
 
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
index 3c48dbb..9805641 100644
--- 
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
+++ 
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java
@@ -21,6 +21,7 @@ package org.apache.hive.service.cli.thrift;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIServiceClient;
 import org.apache.hive.service.cli.FetchOrientation;
@@ -83,6 +84,8 @@ import org.apache.hive.service.rpc.thrift.TStatus;
 import org.apache.hive.service.rpc.thrift.TStatusCode;
 import org.apache.thrift.TException;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * ThriftCLIServiceClient.
  *
@@ -90,10 +93,16 @@ import org.apache.thrift.TException;
 public class ThriftCLIServiceClient extends CLIServiceClient {
   private final TCLIService.Iface cliService;
 
-  public ThriftCLIServiceClient(TCLIService.Iface cliService) {
+  public ThriftCLIServiceClient(TCLIService.Iface cliService, Configuration 
conf) {
+    super(conf);
     this.cliService = cliService;
   }
 
+  @VisibleForTesting
+  public ThriftCLIServiceClient(TCLIService.Iface cliService) {
+    this(cliService, new Configuration());
+  }
+
   public void checkStatus(TStatus status) throws HiveSQLException {
     if (TStatusCode.ERROR_STATUS.equals(status.getStatusCode())) {
       throw new HiveSQLException(status);
@@ -453,8 +462,8 @@ public class ThriftCLIServiceClient extends 
CLIServiceClient {
    */
   @Override
   public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException 
{
-    // TODO: set the correct default fetch size
-    return fetchResults(opHandle, FetchOrientation.FETCH_NEXT, 10000, 
FetchType.QUERY_OUTPUT);
+    return fetchResults(
+        opHandle, FetchOrientation.FETCH_NEXT, defaultFetchRows, 
FetchType.QUERY_OUTPUT);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/47d90467/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java
----------------------------------------------------------------------
diff --git 
a/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java
 
b/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java
index a167b3e..2855bb3 100644
--- 
a/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java
+++ 
b/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java
@@ -97,7 +97,7 @@ public class TestRetryingThriftCLIServiceClient {
       ICLIService cliService =
         (ICLIService) 
Proxy.newProxyInstance(RetryingThriftCLIServiceClientTest.class.getClassLoader(),
           CLIServiceClient.class.getInterfaces(), handlerInst);
-      return new CLIServiceClientWrapper(cliService, tTransport);
+      return new CLIServiceClientWrapper(cliService, tTransport, conf);
     }
 
     @Override

Reply via email to