Repository: hive Updated Branches: refs/heads/master 1c970d924 -> da13a1379
HIVE-14388: Add number of rows inserted message after insert command in Beeline (Bharathkrishna Guruvayoor Murali, reviewed by Sahil Takiar, Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da13a137 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da13a137 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da13a137 Branch: refs/heads/master Commit: da13a13797c7d9926f9a2332ee7b33c7b5f08a1e Parents: 1c970d9 Author: Bharathkrishna Guruvayoor Murali <bhar...@cloudera.com> Authored: Fri Jun 1 05:46:54 2018 -0500 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Fri Jun 1 05:46:54 2018 -0500 ---------------------------------------------------------------------- beeline/src/main/resources/BeeLine.properties | 2 +- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 36 ++++--- .../org/apache/hive/jdbc/HiveStatement.java | 16 ++- .../java/org/apache/hadoop/hive/ql/Driver.java | 15 +++ .../org/apache/hadoop/hive/ql/MapRedStats.java | 10 ++ .../org/apache/hadoop/hive/ql/QueryState.java | 12 +++ .../hadoop/hive/ql/exec/FileSinkOperator.java | 23 ++-- .../hive/ql/exec/mr/HadoopJobExecHelper.java | 11 ++ .../clientpositive/llap/dp_counter_mm.q.out | 8 ++ .../clientpositive/llap/dp_counter_non_mm.q.out | 8 ++ .../clientpositive/llap/orc_ppd_basic.q.out | 1 + .../llap/tez_input_counters.q.out | 1 + service-rpc/if/TCLIService.thrift | 1 + .../gen/thrift/gen-cpp/TCLIService_types.cpp | 22 ++++ .../src/gen/thrift/gen-cpp/TCLIService_types.h | 12 ++- .../rpc/thrift/TGetOperationStatusResp.java | 107 ++++++++++++++++++- service-rpc/src/gen/thrift/gen-php/Types.php | 23 ++++ .../src/gen/thrift/gen-py/TCLIService/ttypes.py | 15 ++- .../gen/thrift/gen-rb/t_c_l_i_service_types.rb | 4 +- .../org/apache/hive/service/cli/CLIService.java | 2 + .../hive/service/cli/OperationStatus.java | 9 ++ .../hive/service/cli/operation/Operation.java | 4 + .../service/cli/thrift/ThriftCLIService.java | 6 ++ 23 files changed, 303 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/beeline/src/main/resources/BeeLine.properties ---------------------------------------------------------------------- diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties index c41b3ed..30b4ef9 100644 --- a/beeline/src/main/resources/BeeLine.properties +++ b/beeline/src/main/resources/BeeLine.properties @@ -138,7 +138,7 @@ abort-drop-all: Aborting drop all tables. drivers-found-count: 0#No driver classes found|1#{0} driver class found|1<{0} driver classes found rows-selected: 0#No rows selected|1#{0} row selected|1<{0} rows selected -rows-affected: 0#No rows affected|1#{0} row affected|1<{0} rows affected|0>Unknown rows affected +rows-affected: -1#Unknown rows affected|0#No rows affected|1#{0} row affected|1<{0} rows affected active-connections: 0#No active connections|1#{0} active connection:|1<{0} active connections: time-ms: ({0,number,#.###} seconds) http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index b217259..d47c136 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -977,7 +977,7 @@ public class TestJdbcDriver2 { assertNotNull("ResultSet is null", res); assertTrue("getResultSet() not returning expected ResultSet", res == stmt .getResultSet()); - assertEquals("get update count not as expected", -1, stmt.getUpdateCount()); + assertEquals("get update count not as expected", 0, stmt.getUpdateCount()); int i = 0; ResultSetMetaData meta = res.getMetaData(); @@ -2962,23 +2962,25 @@ public class TestJdbcDriver2 { private void testInsertOverwrite(HiveStatement stmt) throws SQLException { String tblName = "testInsertOverwriteExecAsync"; - int rowCount = 0; - stmt.execute("create table " + tblName + " (col1 int , col2 string)"); - boolean isResulSet = - stmt.executeAsync("insert overwrite table " + tblName + " select * from " + tableName); - assertFalse(isResulSet); - // HiveStatement#getUpdateCount blocks until the async query is complete - stmt.getUpdateCount(); - // Read from the new table - ResultSet rs = stmt.executeQuery("select * from " + tblName); - assertNotNull(rs); - while (rs.next()) { - String value = rs.getString(2); - rowCount++; - assertNotNull(value); + try { + int rowCount = 0; + stmt.execute("create table " + tblName + " (col1 int , col2 string)"); + boolean isResulSet = + stmt.executeAsync("insert overwrite table " + tblName + " select * from " + tableName); + assertFalse(isResulSet); + // HiveStatement#getUpdateCount blocks until the async query is complete + rowCount = stmt.getUpdateCount(); + // Read from the new table + ResultSet rs = stmt.executeQuery("select * from " + tblName); + assertNotNull(rs); + while (rs.next()) { + String value = rs.getString(2); + assertNotNull(value); + } + assertEquals(dataFileRowCount, rowCount); + } finally { + stmt.execute("drop table " + tblName); } - assertEquals(rowCount, dataFileRowCount); - stmt.execute("drop table " + tblName); } // Test that opening a JDBC connection to a non-existent database throws a HiveSQLException http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java ---------------------------------------------------------------------- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java index 06542ce..ad8d1a7 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java @@ -369,7 +369,7 @@ public class HiveStatement implements java.sql.Statement { TGetOperationStatusResp statusResp = null; // Poll on the operation status, till the operation is complete - while (!isOperationComplete) { + do { try { /** * For an async SQLOperation, GetOperationStatus will use the long polling approach It will @@ -414,7 +414,7 @@ public class HiveStatement implements java.sql.Statement { isLogBeingGenerated = false; throw new SQLException(e.toString(), "08S01", e); } - } + } while (!isOperationComplete); /* we set progress bar to be completed when hive query execution has completed @@ -708,8 +708,16 @@ public class HiveStatement implements java.sql.Statement { * client might end up using executeAsync and then call this to check if the query run is * finished. */ - waitForOperationToComplete(); - return -1; + long numModifiedRows = -1; + TGetOperationStatusResp resp = waitForOperationToComplete(); + if (resp != null) { + numModifiedRows = resp.getNumModifiedRows(); + } + if (numModifiedRows == -1 || numModifiedRows > Integer.MAX_VALUE) { + LOG.warn("Number of rows is greater than Integer.MAX_VALUE"); + return -1; + } + return (int) numModifiedRows; } /* http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 52799b3..7d5e7d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -2326,11 +2326,18 @@ public class Driver implements IDriver { Map<String, MapRedStats> stats = SessionState.get().getMapRedStats(); if (stats != null && !stats.isEmpty()) { long totalCpu = 0; + long numModifiedRows = 0; console.printInfo("MapReduce Jobs Launched: "); for (Map.Entry<String, MapRedStats> entry : stats.entrySet()) { console.printInfo("Stage-" + entry.getKey() + ": " + entry.getValue()); totalCpu += entry.getValue().getCpuMSec(); + + if (numModifiedRows > -1) { + //if overflow, then numModifiedRows is set as -1. Else update numModifiedRows with the sum. + numModifiedRows = addWithOverflowCheck(numModifiedRows, entry.getValue().getNumModifiedRows()); + } } + queryState.setNumModifiedRows(numModifiedRows); console.printInfo("Total MapReduce CPU Time Spent: " + Utilities.formatMsecToStr(totalCpu)); } lDrvState.stateLock.lock(); @@ -2351,6 +2358,14 @@ public class Driver implements IDriver { } } + private long addWithOverflowCheck(long val1, long val2) { + try { + return Math.addExact(val1, val2); + } catch (ArithmeticException e) { + return -1; + } + } + private void releasePlan(QueryPlan plan) { // Plan maybe null if Driver.close is called in another thread for the same Driver object lDrvState.stateLock.lock(); http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/java/org/apache/hadoop/hive/ql/MapRedStats.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/MapRedStats.java b/ql/src/java/org/apache/hadoop/hive/ql/MapRedStats.java index cf9c227..483c3d9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/MapRedStats.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/MapRedStats.java @@ -38,6 +38,8 @@ public class MapRedStats { String jobId; + private long numModifiedRows; + public MapRedStats(int numMap, int numReduce, long cpuMSec, boolean ifSuccess, String jobId) { this.numMap = numMap; this.numReduce = numReduce; @@ -94,6 +96,14 @@ public class MapRedStats { this.jobId = jobId; } + public long getNumModifiedRows() { + return numModifiedRows; + } + + public void setNumModifiedRows(long numModifiedRows) { + this.numModifiedRows = numModifiedRows; + } + public String getTaskNumbers() { StringBuilder sb = new StringBuilder(); if (numMap > 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java index 706c9ff..b1a602c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryState.java @@ -50,6 +50,11 @@ public class QueryState { private HiveTxnManager txnManager; /** + * Holds the number of rows affected for insert queries. + */ + private long numModifiedRows = 0; + + /** * Private constructor, use QueryState.Builder instead. * @param conf The query specific configuration object */ @@ -100,6 +105,13 @@ public class QueryState { this.txnManager = txnManager; } + public long getNumModifiedRows() { + return numModifiedRows; + } + + public void setNumModifiedRows(long numModifiedRows) { + this.numModifiedRows = numModifiedRows; + } /** * Builder to instantiate the QueryState object. */ http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 9c57eff..83b53f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE import java.io.IOException; import java.io.Serializable; -import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -83,27 +82,13 @@ import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.Murmur3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; import java.util.function.BiFunction; - -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE_STORAGE; - /** * File Sink operator implementation. **/ @@ -147,6 +132,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements private transient boolean isInsertOverwrite; private transient String counterGroup; private transient BiFunction<Object[], ObjectInspector[], Integer> hashFunc; + public static final String TOTAL_TABLE_ROWS_WRITTEN = "TOTAL_TABLE_ROWS_WRITTEN"; + /** * Counters. */ @@ -583,6 +570,12 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements hashFunc = conf.getTableInfo().getBucketingVersion() == 2 ? ObjectInspectorUtils::getBucketHashCode : ObjectInspectorUtils::getBucketHashCodeOld; + + //Counter for number of rows that are associated with a destination table in FileSinkOperator. + //This count is used to get total number of rows in an insert query. + if (conf.getTableInfo() != null && conf.getTableInfo().getTableName() != null) { + statsMap.put(TOTAL_TABLE_ROWS_WRITTEN, row_count); + } } catch (HiveException e) { throw e; } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java index fcdc996..c31e22f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.MapRedStats; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskHandle; @@ -426,6 +427,16 @@ public class HadoopJobExecHelper { SessionState ss = SessionState.get(); if (ss != null) { + //Set the number of table rows affected in mapRedStats to display number of rows inserted. + if (ctrs != null) { + Counter counter = ctrs.findCounter( + ss.getConf().getVar(HiveConf.ConfVars.HIVECOUNTERGROUP), + FileSinkOperator.TOTAL_TABLE_ROWS_WRITTEN); + if (counter != null) { + mapRedStats.setNumModifiedRows(counter.getValue()); + } + } + this.callBackObj.logPlanProgress(ss); } // LOG.info(queryPlan); http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out index 1be25fe..25dc0c9 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out @@ -35,6 +35,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 84 RECORDS_OUT_OPERATOR_SEL_5: 57 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 84 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -65,6 +66,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 189 RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -103,6 +105,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 189 RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -133,6 +136,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 292 RECORDS_OUT_OPERATOR_SEL_5: 184 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -185,6 +189,7 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_SEL_6: 105 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -223,6 +228,7 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_5: 184 RECORDS_OUT_OPERATOR_SEL_6: 208 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -270,6 +276,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_7: 189 RECORDS_OUT_OPERATOR_TS_0: 500 RECORDS_OUT_OPERATOR_TS_3: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -313,6 +320,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_7: 292 RECORDS_OUT_OPERATOR_TS_0: 500 RECORDS_OUT_OPERATOR_TS_3: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out index c0d2141..bbe4044 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out @@ -35,6 +35,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 84 RECORDS_OUT_OPERATOR_SEL_5: 57 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 84 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -65,6 +66,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 189 RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -103,6 +105,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 189 RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -133,6 +136,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_2: 292 RECORDS_OUT_OPERATOR_SEL_5: 184 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -185,6 +189,7 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_5: 121 RECORDS_OUT_OPERATOR_SEL_6: 105 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -223,6 +228,7 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_5: 184 RECORDS_OUT_OPERATOR_SEL_6: 208 RECORDS_OUT_OPERATOR_TS_0: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -270,6 +276,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_7: 189 RECORDS_OUT_OPERATOR_TS_0: 500 RECORDS_OUT_OPERATOR_TS_3: 500 + TOTAL_TABLE_ROWS_WRITTEN: 189 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -313,6 +320,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_7: 292 RECORDS_OUT_OPERATOR_TS_0: 500 RECORDS_OUT_OPERATOR_TS_3: 500 + TOTAL_TABLE_ROWS_WRITTEN: 292 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 49f9544..ad8aef0 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -1878,6 +1878,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_MAP_0: 0 RECORDS_OUT_OPERATOR_SEL_2: 2 RECORDS_OUT_OPERATOR_TS_0: 98 + TOTAL_TABLE_ROWS_WRITTEN: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 2359296 ALLOCATED_USED_BYTES: 44166 http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out index d2fcdaa..63fdd74 100644 --- a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out @@ -1829,6 +1829,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_SEL_5: 240 RECORDS_OUT_OPERATOR_SEL_6: 240 RECORDS_OUT_OPERATOR_TS_0: 240 + TOTAL_TABLE_ROWS_WRITTEN: 240 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 922 NUM_DECODED_BATCHES: 148 http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/if/TCLIService.thrift ---------------------------------------------------------------------- diff --git a/service-rpc/if/TCLIService.thrift b/service-rpc/if/TCLIService.thrift index 30f8af7..1cf8a91 100644 --- a/service-rpc/if/TCLIService.thrift +++ b/service-rpc/if/TCLIService.thrift @@ -1070,6 +1070,7 @@ struct TGetOperationStatusResp { 10: optional TProgressUpdateResp progressUpdateResponse + 11: optional i64 numModifiedRows } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.cpp ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.cpp b/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.cpp index b2b62c7..d312896 100644 --- a/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.cpp +++ b/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.cpp @@ -8571,6 +8571,11 @@ void TGetOperationStatusResp::__set_progressUpdateResponse(const TProgressUpdate __isset.progressUpdateResponse = true; } +void TGetOperationStatusResp::__set_numModifiedRows(const int64_t val) { + this->numModifiedRows = val; +__isset.numModifiedRows = true; +} + uint32_t TGetOperationStatusResp::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -8675,6 +8680,14 @@ uint32_t TGetOperationStatusResp::read(::apache::thrift::protocol::TProtocol* ip xfer += iprot->skip(ftype); } break; + case 11: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->numModifiedRows); + this->__isset.numModifiedRows = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -8743,6 +8756,11 @@ uint32_t TGetOperationStatusResp::write(::apache::thrift::protocol::TProtocol* o xfer += this->progressUpdateResponse.write(oprot); xfer += oprot->writeFieldEnd(); } + if (this->__isset.numModifiedRows) { + xfer += oprot->writeFieldBegin("numModifiedRows", ::apache::thrift::protocol::T_I64, 11); + xfer += oprot->writeI64(this->numModifiedRows); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -8760,6 +8778,7 @@ void swap(TGetOperationStatusResp &a, TGetOperationStatusResp &b) { swap(a.operationCompleted, b.operationCompleted); swap(a.hasResultSet, b.hasResultSet); swap(a.progressUpdateResponse, b.progressUpdateResponse); + swap(a.numModifiedRows, b.numModifiedRows); swap(a.__isset, b.__isset); } @@ -8774,6 +8793,7 @@ TGetOperationStatusResp::TGetOperationStatusResp(const TGetOperationStatusResp& operationCompleted = other283.operationCompleted; hasResultSet = other283.hasResultSet; progressUpdateResponse = other283.progressUpdateResponse; + numModifiedRows = other283.numModifiedRows; __isset = other283.__isset; } TGetOperationStatusResp& TGetOperationStatusResp::operator=(const TGetOperationStatusResp& other284) { @@ -8787,6 +8807,7 @@ TGetOperationStatusResp& TGetOperationStatusResp::operator=(const TGetOperationS operationCompleted = other284.operationCompleted; hasResultSet = other284.hasResultSet; progressUpdateResponse = other284.progressUpdateResponse; + numModifiedRows = other284.numModifiedRows; __isset = other284.__isset; return *this; } @@ -8803,6 +8824,7 @@ void TGetOperationStatusResp::printTo(std::ostream& out) const { out << ", " << "operationCompleted="; (__isset.operationCompleted ? (out << to_string(operationCompleted)) : (out << "<null>")); out << ", " << "hasResultSet="; (__isset.hasResultSet ? (out << to_string(hasResultSet)) : (out << "<null>")); out << ", " << "progressUpdateResponse="; (__isset.progressUpdateResponse ? (out << to_string(progressUpdateResponse)) : (out << "<null>")); + out << ", " << "numModifiedRows="; (__isset.numModifiedRows ? (out << to_string(numModifiedRows)) : (out << "<null>")); out << ")"; } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.h ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.h b/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.h index 4321ad6..b30c28f 100644 --- a/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.h +++ b/service-rpc/src/gen/thrift/gen-cpp/TCLIService_types.h @@ -3840,7 +3840,7 @@ inline std::ostream& operator<<(std::ostream& out, const TGetOperationStatusReq& } typedef struct _TGetOperationStatusResp__isset { - _TGetOperationStatusResp__isset() : operationState(false), sqlState(false), errorCode(false), errorMessage(false), taskStatus(false), operationStarted(false), operationCompleted(false), hasResultSet(false), progressUpdateResponse(false) {} + _TGetOperationStatusResp__isset() : operationState(false), sqlState(false), errorCode(false), errorMessage(false), taskStatus(false), operationStarted(false), operationCompleted(false), hasResultSet(false), progressUpdateResponse(false), numModifiedRows(false) {} bool operationState :1; bool sqlState :1; bool errorCode :1; @@ -3850,6 +3850,7 @@ typedef struct _TGetOperationStatusResp__isset { bool operationCompleted :1; bool hasResultSet :1; bool progressUpdateResponse :1; + bool numModifiedRows :1; } _TGetOperationStatusResp__isset; class TGetOperationStatusResp { @@ -3857,7 +3858,7 @@ class TGetOperationStatusResp { TGetOperationStatusResp(const TGetOperationStatusResp&); TGetOperationStatusResp& operator=(const TGetOperationStatusResp&); - TGetOperationStatusResp() : operationState((TOperationState::type)0), sqlState(), errorCode(0), errorMessage(), taskStatus(), operationStarted(0), operationCompleted(0), hasResultSet(0) { + TGetOperationStatusResp() : operationState((TOperationState::type)0), sqlState(), errorCode(0), errorMessage(), taskStatus(), operationStarted(0), operationCompleted(0), hasResultSet(0), numModifiedRows(0) { } virtual ~TGetOperationStatusResp() throw(); @@ -3871,6 +3872,7 @@ class TGetOperationStatusResp { int64_t operationCompleted; bool hasResultSet; TProgressUpdateResp progressUpdateResponse; + int64_t numModifiedRows; _TGetOperationStatusResp__isset __isset; @@ -3894,6 +3896,8 @@ class TGetOperationStatusResp { void __set_progressUpdateResponse(const TProgressUpdateResp& val); + void __set_numModifiedRows(const int64_t val); + bool operator == (const TGetOperationStatusResp & rhs) const { if (!(status == rhs.status)) @@ -3934,6 +3938,10 @@ class TGetOperationStatusResp { return false; else if (__isset.progressUpdateResponse && !(progressUpdateResponse == rhs.progressUpdateResponse)) return false; + if (__isset.numModifiedRows != rhs.__isset.numModifiedRows) + return false; + else if (__isset.numModifiedRows && !(numModifiedRows == rhs.numModifiedRows)) + return false; return true; } bool operator != (const TGetOperationStatusResp &rhs) const { http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java index 15e8220..a7c1f86 100644 --- a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java +++ b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java @@ -48,6 +48,7 @@ import org.slf4j.LoggerFactory; private static final org.apache.thrift.protocol.TField OPERATION_COMPLETED_FIELD_DESC = new org.apache.thrift.protocol.TField("operationCompleted", org.apache.thrift.protocol.TType.I64, (short)8); private static final org.apache.thrift.protocol.TField HAS_RESULT_SET_FIELD_DESC = new org.apache.thrift.protocol.TField("hasResultSet", org.apache.thrift.protocol.TType.BOOL, (short)9); private static final org.apache.thrift.protocol.TField PROGRESS_UPDATE_RESPONSE_FIELD_DESC = new org.apache.thrift.protocol.TField("progressUpdateResponse", org.apache.thrift.protocol.TType.STRUCT, (short)10); + private static final org.apache.thrift.protocol.TField NUM_MODIFIED_ROWS_FIELD_DESC = new org.apache.thrift.protocol.TField("numModifiedRows", org.apache.thrift.protocol.TType.I64, (short)11); private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); static { @@ -65,6 +66,7 @@ import org.slf4j.LoggerFactory; private long operationCompleted; // optional private boolean hasResultSet; // optional private TProgressUpdateResp progressUpdateResponse; // optional + private long numModifiedRows; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -81,7 +83,8 @@ import org.slf4j.LoggerFactory; OPERATION_STARTED((short)7, "operationStarted"), OPERATION_COMPLETED((short)8, "operationCompleted"), HAS_RESULT_SET((short)9, "hasResultSet"), - PROGRESS_UPDATE_RESPONSE((short)10, "progressUpdateResponse"); + PROGRESS_UPDATE_RESPONSE((short)10, "progressUpdateResponse"), + NUM_MODIFIED_ROWS((short)11, "numModifiedRows"); private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); @@ -116,6 +119,8 @@ import org.slf4j.LoggerFactory; return HAS_RESULT_SET; case 10: // PROGRESS_UPDATE_RESPONSE return PROGRESS_UPDATE_RESPONSE; + case 11: // NUM_MODIFIED_ROWS + return NUM_MODIFIED_ROWS; default: return null; } @@ -160,8 +165,9 @@ import org.slf4j.LoggerFactory; private static final int __OPERATIONSTARTED_ISSET_ID = 1; private static final int __OPERATIONCOMPLETED_ISSET_ID = 2; private static final int __HASRESULTSET_ISSET_ID = 3; + private static final int __NUMMODIFIEDROWS_ISSET_ID = 4; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.OPERATION_STATE,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE,_Fields.TASK_STATUS,_Fields.OPERATION_STARTED,_Fields.OPERATION_COMPLETED,_Fields.HAS_RESULT_SET,_Fields.PROGRESS_UPDATE_RESPONSE}; + private static final _Fields optionals[] = {_Fields.OPERATION_STATE,_Fields.SQL_STATE,_Fields.ERROR_CODE,_Fields.ERROR_MESSAGE,_Fields.TASK_STATUS,_Fields.OPERATION_STARTED,_Fields.OPERATION_COMPLETED,_Fields.HAS_RESULT_SET,_Fields.PROGRESS_UPDATE_RESPONSE,_Fields.NUM_MODIFIED_ROWS}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -185,6 +191,8 @@ import org.slf4j.LoggerFactory; new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); tmpMap.put(_Fields.PROGRESS_UPDATE_RESPONSE, new org.apache.thrift.meta_data.FieldMetaData("progressUpdateResponse", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "TProgressUpdateResp"))); + tmpMap.put(_Fields.NUM_MODIFIED_ROWS, new org.apache.thrift.meta_data.FieldMetaData("numModifiedRows", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TGetOperationStatusResp.class, metaDataMap); } @@ -226,6 +234,7 @@ import org.slf4j.LoggerFactory; if (other.isSetProgressUpdateResponse()) { this.progressUpdateResponse = other.progressUpdateResponse; } + this.numModifiedRows = other.numModifiedRows; } public TGetOperationStatusResp deepCopy() { @@ -248,6 +257,8 @@ import org.slf4j.LoggerFactory; setHasResultSetIsSet(false); this.hasResultSet = false; this.progressUpdateResponse = null; + setNumModifiedRowsIsSet(false); + this.numModifiedRows = 0; } public TStatus getStatus() { @@ -484,6 +495,28 @@ import org.slf4j.LoggerFactory; } } + public long getNumModifiedRows() { + return this.numModifiedRows; + } + + public void setNumModifiedRows(long numModifiedRows) { + this.numModifiedRows = numModifiedRows; + setNumModifiedRowsIsSet(true); + } + + public void unsetNumModifiedRows() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __NUMMODIFIEDROWS_ISSET_ID); + } + + /** Returns true if field numModifiedRows is set (has been assigned a value) and false otherwise */ + public boolean isSetNumModifiedRows() { + return EncodingUtils.testBit(__isset_bitfield, __NUMMODIFIEDROWS_ISSET_ID); + } + + public void setNumModifiedRowsIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMMODIFIEDROWS_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case STATUS: @@ -566,6 +599,14 @@ import org.slf4j.LoggerFactory; } break; + case NUM_MODIFIED_ROWS: + if (value == null) { + unsetNumModifiedRows(); + } else { + setNumModifiedRows((Long)value); + } + break; + } } @@ -601,6 +642,9 @@ import org.slf4j.LoggerFactory; case PROGRESS_UPDATE_RESPONSE: return getProgressUpdateResponse(); + case NUM_MODIFIED_ROWS: + return getNumModifiedRows(); + } throw new IllegalStateException(); } @@ -632,6 +676,8 @@ import org.slf4j.LoggerFactory; return isSetHasResultSet(); case PROGRESS_UPDATE_RESPONSE: return isSetProgressUpdateResponse(); + case NUM_MODIFIED_ROWS: + return isSetNumModifiedRows(); } throw new IllegalStateException(); } @@ -739,6 +785,15 @@ import org.slf4j.LoggerFactory; return false; } + boolean this_present_numModifiedRows = true && this.isSetNumModifiedRows(); + boolean that_present_numModifiedRows = true && that.isSetNumModifiedRows(); + if (this_present_numModifiedRows || that_present_numModifiedRows) { + if (!(this_present_numModifiedRows && that_present_numModifiedRows)) + return false; + if (this.numModifiedRows != that.numModifiedRows) + return false; + } + return true; } @@ -796,6 +851,11 @@ import org.slf4j.LoggerFactory; if (present_progressUpdateResponse) list.add(progressUpdateResponse); + boolean present_numModifiedRows = true && (isSetNumModifiedRows()); + list.add(present_numModifiedRows); + if (present_numModifiedRows) + list.add(numModifiedRows); + return list.hashCode(); } @@ -907,6 +967,16 @@ import org.slf4j.LoggerFactory; return lastComparison; } } + lastComparison = Boolean.valueOf(isSetNumModifiedRows()).compareTo(other.isSetNumModifiedRows()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNumModifiedRows()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.numModifiedRows, other.numModifiedRows); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1008,6 +1078,12 @@ import org.slf4j.LoggerFactory; } first = false; } + if (isSetNumModifiedRows()) { + if (!first) sb.append(", "); + sb.append("numModifiedRows:"); + sb.append(this.numModifiedRows); + first = false; + } sb.append(")"); return sb.toString(); } @@ -1142,6 +1218,14 @@ import org.slf4j.LoggerFactory; org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 11: // NUM_MODIFIED_ROWS + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.numModifiedRows = iprot.readI64(); + struct.setNumModifiedRowsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1215,6 +1299,11 @@ import org.slf4j.LoggerFactory; oprot.writeFieldEnd(); } } + if (struct.isSetNumModifiedRows()) { + oprot.writeFieldBegin(NUM_MODIFIED_ROWS_FIELD_DESC); + oprot.writeI64(struct.numModifiedRows); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1261,7 +1350,10 @@ import org.slf4j.LoggerFactory; if (struct.isSetProgressUpdateResponse()) { optionals.set(8); } - oprot.writeBitSet(optionals, 9); + if (struct.isSetNumModifiedRows()) { + optionals.set(9); + } + oprot.writeBitSet(optionals, 10); if (struct.isSetOperationState()) { oprot.writeI32(struct.operationState.getValue()); } @@ -1289,6 +1381,9 @@ import org.slf4j.LoggerFactory; if (struct.isSetProgressUpdateResponse()) { struct.progressUpdateResponse.write(oprot); } + if (struct.isSetNumModifiedRows()) { + oprot.writeI64(struct.numModifiedRows); + } } @Override @@ -1297,7 +1392,7 @@ import org.slf4j.LoggerFactory; struct.status = new TStatus(); struct.status.read(iprot); struct.setStatusIsSet(true); - BitSet incoming = iprot.readBitSet(9); + BitSet incoming = iprot.readBitSet(10); if (incoming.get(0)) { struct.operationState = org.apache.hive.service.rpc.thrift.TOperationState.findByValue(iprot.readI32()); struct.setOperationStateIsSet(true); @@ -1335,6 +1430,10 @@ import org.slf4j.LoggerFactory; struct.progressUpdateResponse.read(iprot); struct.setProgressUpdateResponseIsSet(true); } + if (incoming.get(9)) { + struct.numModifiedRows = iprot.readI64(); + struct.setNumModifiedRowsIsSet(true); + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-php/Types.php ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-php/Types.php b/service-rpc/src/gen/thrift/gen-php/Types.php index abb7c1f..39572d4 100644 --- a/service-rpc/src/gen/thrift/gen-php/Types.php +++ b/service-rpc/src/gen/thrift/gen-php/Types.php @@ -8351,6 +8351,10 @@ class TGetOperationStatusResp { * @var \TProgressUpdateResp */ public $progressUpdateResponse = null; + /** + * @var int + */ + public $numModifiedRows = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -8397,6 +8401,10 @@ class TGetOperationStatusResp { 'type' => TType::STRUCT, 'class' => '\TProgressUpdateResp', ), + 11 => array( + 'var' => 'numModifiedRows', + 'type' => TType::I64, + ), ); } if (is_array($vals)) { @@ -8430,6 +8438,9 @@ class TGetOperationStatusResp { if (isset($vals['progressUpdateResponse'])) { $this->progressUpdateResponse = $vals['progressUpdateResponse']; } + if (isset($vals['numModifiedRows'])) { + $this->numModifiedRows = $vals['numModifiedRows']; + } } } @@ -8524,6 +8535,13 @@ class TGetOperationStatusResp { $xfer += $input->skip($ftype); } break; + case 11: + if ($ftype == TType::I64) { + $xfer += $input->readI64($this->numModifiedRows); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -8593,6 +8611,11 @@ class TGetOperationStatusResp { $xfer += $this->progressUpdateResponse->write($output); $xfer += $output->writeFieldEnd(); } + if ($this->numModifiedRows !== null) { + $xfer += $output->writeFieldBegin('numModifiedRows', TType::I64, 11); + $xfer += $output->writeI64($this->numModifiedRows); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-py/TCLIService/ttypes.py ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-py/TCLIService/ttypes.py b/service-rpc/src/gen/thrift/gen-py/TCLIService/ttypes.py index 0f8fd07..989ba44 100644 --- a/service-rpc/src/gen/thrift/gen-py/TCLIService/ttypes.py +++ b/service-rpc/src/gen/thrift/gen-py/TCLIService/ttypes.py @@ -6303,6 +6303,7 @@ class TGetOperationStatusResp: - operationCompleted - hasResultSet - progressUpdateResponse + - numModifiedRows """ thrift_spec = ( @@ -6317,9 +6318,10 @@ class TGetOperationStatusResp: (8, TType.I64, 'operationCompleted', None, None, ), # 8 (9, TType.BOOL, 'hasResultSet', None, None, ), # 9 (10, TType.STRUCT, 'progressUpdateResponse', (TProgressUpdateResp, TProgressUpdateResp.thrift_spec), None, ), # 10 + (11, TType.I64, 'numModifiedRows', None, None, ), # 11 ) - def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None,): + def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None,): self.status = status self.operationState = operationState self.sqlState = sqlState @@ -6330,6 +6332,7 @@ class TGetOperationStatusResp: self.operationCompleted = operationCompleted self.hasResultSet = hasResultSet self.progressUpdateResponse = progressUpdateResponse + self.numModifiedRows = numModifiedRows def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -6392,6 +6395,11 @@ class TGetOperationStatusResp: self.progressUpdateResponse.read(iprot) else: iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I64: + self.numModifiedRows = iprot.readI64() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -6442,6 +6450,10 @@ class TGetOperationStatusResp: oprot.writeFieldBegin('progressUpdateResponse', TType.STRUCT, 10) self.progressUpdateResponse.write(oprot) oprot.writeFieldEnd() + if self.numModifiedRows is not None: + oprot.writeFieldBegin('numModifiedRows', TType.I64, 11) + oprot.writeI64(self.numModifiedRows) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -6463,6 +6475,7 @@ class TGetOperationStatusResp: value = (value * 31) ^ hash(self.operationCompleted) value = (value * 31) ^ hash(self.hasResultSet) value = (value * 31) ^ hash(self.progressUpdateResponse) + value = (value * 31) ^ hash(self.numModifiedRows) return value def __repr__(self): http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service-rpc/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb ---------------------------------------------------------------------- diff --git a/service-rpc/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb b/service-rpc/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb index 60183da..6ab4063 100644 --- a/service-rpc/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb +++ b/service-rpc/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb @@ -1624,6 +1624,7 @@ class TGetOperationStatusResp OPERATIONCOMPLETED = 8 HASRESULTSET = 9 PROGRESSUPDATERESPONSE = 10 + NUMMODIFIEDROWS = 11 FIELDS = { STATUS => {:type => ::Thrift::Types::STRUCT, :name => 'status', :class => ::TStatus}, @@ -1635,7 +1636,8 @@ class TGetOperationStatusResp OPERATIONSTARTED => {:type => ::Thrift::Types::I64, :name => 'operationStarted', :optional => true}, OPERATIONCOMPLETED => {:type => ::Thrift::Types::I64, :name => 'operationCompleted', :optional => true}, HASRESULTSET => {:type => ::Thrift::Types::BOOL, :name => 'hasResultSet', :optional => true}, - PROGRESSUPDATERESPONSE => {:type => ::Thrift::Types::STRUCT, :name => 'progressUpdateResponse', :class => ::TProgressUpdateResp, :optional => true} + PROGRESSUPDATERESPONSE => {:type => ::Thrift::Types::STRUCT, :name => 'progressUpdateResponse', :class => ::TProgressUpdateResp, :optional => true}, + NUMMODIFIEDROWS => {:type => ::Thrift::Types::I64, :name => 'numModifiedRows', :optional => true} } def struct_fields; FIELDS; end http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service/src/java/org/apache/hive/service/cli/CLIService.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java index 3e26197..dbfaf71 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIService.java +++ b/service/src/java/org/apache/hive/service/cli/CLIService.java @@ -474,6 +474,8 @@ public class CLIService extends CompositeService implements ICLIService { } OperationStatus opStatus = operation.getStatus(); LOG.debug(opHandle + ": getOperationStatus()"); + long numModifiedRows = operation.getNumModifiedRows(); + opStatus.setNumModifiedRows(numModifiedRows); opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation, conf)); return opStatus; } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service/src/java/org/apache/hive/service/cli/OperationStatus.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/OperationStatus.java b/service/src/java/org/apache/hive/service/cli/OperationStatus.java index 52cc3ae..581bf92 100644 --- a/service/src/java/org/apache/hive/service/cli/OperationStatus.java +++ b/service/src/java/org/apache/hive/service/cli/OperationStatus.java @@ -31,6 +31,7 @@ public class OperationStatus { private final boolean hasResultSet; private final HiveSQLException operationException; private JobProgressUpdate jobProgressUpdate; + private long numModifiedRows; public OperationStatus(OperationState state, String taskStatus, long operationStarted, long operationCompleted, boolean hasResultSet, HiveSQLException operationException) { this.state = state; @@ -72,4 +73,12 @@ public class OperationStatus { public JobProgressUpdate jobProgressUpdate(){ return jobProgressUpdate; } + + public long getNumModifiedRows() { + return numModifiedRows; + } + + void setNumModifiedRows(long numRows) { + this.numModifiedRows = numRows; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service/src/java/org/apache/hive/service/cli/operation/Operation.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 3706c72..07ab487 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -184,6 +184,10 @@ public abstract class Operation { this.operationTimeout = operationTimeout; } + public long getNumModifiedRows() { + return queryState.getNumModifiedRows(); + } + protected void setOperationException(HiveSQLException operationException) { this.operationException = operationException; } http://git-wip-us.apache.org/repos/asf/hive/blob/da13a137/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index c64c991..d425423 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -20,6 +20,7 @@ package org.apache.hive.service.cli.thrift; import static com.google.common.base.Preconditions.checkArgument; +import org.apache.hive.service.cli.OperationState; import org.apache.hive.service.rpc.thrift.TSetClientInfoReq; import org.apache.hive.service.rpc.thrift.TSetClientInfoResp; @@ -691,6 +692,11 @@ public abstract class ThriftCLIService extends AbstractService implements TCLISe try { OperationStatus operationStatus = cliService.getOperationStatus(operationHandle, req.isGetProgressUpdate()); + + if (operationStatus.getState().equals(OperationState.FINISHED)) { + long numModifiedRows = operationStatus.getNumModifiedRows(); + resp.setNumModifiedRows(numModifiedRows); + } resp.setOperationState(operationStatus.getState().toTOperationState()); resp.setErrorMessage(operationStatus.getState().getErrorMessage()); HiveSQLException opException = operationStatus.getOperationException();
