Author: nzhang
Date: Fri Mar 18 04:15:37 2011
New Revision: 1082819
URL: http://svn.apache.org/viewvc?rev=1082819&view=rev
Log:
HIVE-1851. The class HiveResultSet should implement batch fetching (Bennie
Schut via Ning Zhang)
Modified:
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveStatement.java
hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
Modified:
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
URL:
http://svn.apache.org/viewvc/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java?rev=1082819&r1=1082818&r2=1082819&view=diff
==============================================================================
---
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
(original)
+++
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
Fri Mar 18 04:15:37 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.jdbc;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import java.util.Properties;
@@ -54,6 +55,10 @@ public class HiveQueryResultSet extends
private int maxRows = 0;
private int rowsFetched = 0;
+ private int fetchSize = 50;
+
+ private List<String> fetchedRows;
+ private Iterator<String> fetchedRowsItr;
public HiveQueryResultSet(HiveInterface client, int maxRows) throws
SQLException {
this.client = client;
@@ -96,18 +101,18 @@ public class HiveQueryResultSet extends
serde = new LazySimpleSerDe();
Properties props = new Properties();
if (names.length() > 0) {
- LOG.info("Column names: " + names);
+ LOG.debug("Column names: " + names);
props.setProperty(Constants.LIST_COLUMNS, names);
}
if (types.length() > 0) {
- LOG.info("Column types: " + types);
+ LOG.debug("Column types: " + types);
props.setProperty(Constants.LIST_COLUMN_TYPES, types);
}
serde.initialize(new Configuration(), props);
} catch (Exception ex) {
ex.printStackTrace();
- throw new SQLException("Could not create ResultSet: " + ex.getMessage());
+ throw new SQLException("Could not create ResultSet: " + ex.getMessage(),
ex);
}
}
@@ -128,9 +133,19 @@ public class HiveQueryResultSet extends
return false;
}
- String rowStr = "";
try {
- rowStr = (String) client.fetchOne();
+ if (fetchedRows == null || !fetchedRowsItr.hasNext()) {
+ fetchedRows = client.fetchN(fetchSize);
+ fetchedRowsItr = fetchedRows.iterator();
+ }
+
+ String rowStr = "";
+ if (fetchedRowsItr.hasNext()) {
+ rowStr = fetchedRowsItr.next();
+ } else {
+ return false;
+ }
+
rowsFetched++;
if (LOG.isDebugEnabled()) {
LOG.debug("Fetched row string: " + rowStr);
@@ -165,6 +180,16 @@ public class HiveQueryResultSet extends
return true;
}
+ @Override
+ public void setFetchSize(int rows) throws SQLException {
+ fetchSize = rows;
+ }
+
+ @Override
+ public int getFetchSize() throws SQLException {
+ return fetchSize;
+ }
+
/**
* Convert a LazyObject to a standard Java object in compliance with JDBC
3.0 (see JDBC 3.0
* Specification, Table B-3: Mapping from JDBC Types to Java Object Types).
Modified:
hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveStatement.java
URL:
http://svn.apache.org/viewvc/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveStatement.java?rev=1082819&r1=1082818&r2=1082819&view=diff
==============================================================================
--- hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveStatement.java
(original)
+++ hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveStatement.java Fri
Mar 18 04:15:37 2011
@@ -33,6 +33,8 @@ import org.apache.hadoop.hive.service.Hi
public class HiveStatement implements java.sql.Statement {
private JdbcSessionState session;
private HiveInterface client;
+ private int fetchSize = 50;
+
/**
* We need to keep a reference to the result set to support the following:
* <code>
@@ -191,6 +193,7 @@ public class HiveStatement implements ja
throw new SQLException(ex.toString(), "08S01");
}
resultSet = new HiveQueryResultSet(client, maxRows);
+ resultSet.setFetchSize(fetchSize);
return resultSet;
}
@@ -266,7 +269,7 @@ public class HiveStatement implements ja
*/
public int getFetchSize() throws SQLException {
- throw new SQLException("Method not supported");
+ return fetchSize;
}
/*
@@ -446,7 +449,7 @@ public class HiveStatement implements ja
*/
public void setFetchSize(int rows) throws SQLException {
- throw new SQLException("Method not supported");
+ fetchSize = rows;
}
/*
Modified:
hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java?rev=1082819&r1=1082818&r2=1082819&view=diff
==============================================================================
--- hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
(original)
+++ hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
Fri Mar 18 04:15:37 2011
@@ -192,19 +192,23 @@ public class TestJdbcDriver extends Test
}
public final void testSelectAll() throws Exception {
- doTestSelectAll(tableName, -1); // tests not setting maxRows (return all)
- doTestSelectAll(tableName, 0); // tests setting maxRows to 0 (return all)
+ doTestSelectAll(tableName, -1, -1); // tests not setting maxRows (return
all)
+ doTestSelectAll(tableName, 0, -1); // tests setting maxRows to 0 (return
all)
}
public final void testSelectAllPartioned() throws Exception {
- doTestSelectAll(partitionedTableName, -1); // tests not setting maxRows
+ doTestSelectAll(partitionedTableName, -1, -1); // tests not setting maxRows
// (return all)
- doTestSelectAll(partitionedTableName, 0); // tests setting maxRows to 0
+ doTestSelectAll(partitionedTableName, 0, -1); // tests setting maxRows to 0
// (return all)
}
public final void testSelectAllMaxRows() throws Exception {
- doTestSelectAll(tableName, 100);
+ doTestSelectAll(tableName, 100, -1);
+ }
+
+ public final void testSelectAllFetchSize() throws Exception {
+ doTestSelectAll(tableName, 100, 20);
}
public void testDataTypes() throws Exception {
@@ -267,11 +271,15 @@ public class TestJdbcDriver extends Test
assertFalse(res.next());
}
- private void doTestSelectAll(String tableName, int maxRows) throws Exception
{
+ private void doTestSelectAll(String tableName, int maxRows, int fetchSize)
throws Exception {
Statement stmt = con.createStatement();
if (maxRows >= 0) {
stmt.setMaxRows(maxRows);
}
+ if (fetchSize > 0) {
+ stmt.setFetchSize(fetchSize);
+ assertEquals(fetchSize, stmt.getFetchSize());
+ }
// JDBC says that 0 means return all, which is the default
int expectedMaxRows = maxRows < 1 ? 0 : maxRows;
