HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Conflicts:
        ql/src/java/org/apache/hadoop/hive/ql/Driver.java


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2474f063
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2474f063
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2474f063

Branch: refs/heads/hive-14535
Commit: 2474f063aaaf3a91fdab9d9c5358723072183ddf
Parents: eacf9f9 36e810f
Author: Sergey Shelukhin <ser...@apache.org>
Authored: Mon Oct 17 12:31:12 2016 -0700
Committer: Sergey Shelukhin <ser...@apache.org>
Committed: Mon Oct 17 12:31:12 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hive/beeline/BeeLine.java   |  176 ++-
 .../java/org/apache/hive/beeline/Commands.java  |   52 +-
 .../BeelineHS2ConnectionFileParseException.java |   30 +
 .../hs2connection/HS2ConnectionFileParser.java  |   88 ++
 .../hs2connection/HS2ConnectionFileUtils.java   |  119 ++
 .../HiveSiteHS2ConnectionFileParser.java        |  172 +++
 .../UserHS2ConnectionFileParser.java            |  117 ++
 .../apache/hive/beeline/TestBeeLineHistory.java |    4 +-
 .../TestUserHS2ConnectionFileParser.java        |  211 ++++
 beeline/src/test/resources/hive-site.xml        |    5 +
 .../test-hs2-conn-conf-kerberos-http.xml        |   48 +
 .../test-hs2-conn-conf-kerberos-nossl.xml       |   32 +
 .../test-hs2-conn-conf-kerberos-ssl.xml         |   40 +
 .../resources/test-hs2-connection-conf-list.xml |   36 +
 .../test-hs2-connection-config-noauth.xml       |   28 +
 .../test-hs2-connection-multi-conf-list.xml     |   37 +
 .../test-hs2-connection-zookeeper-config.xml    |   32 +
 .../apache/hadoop/hive/common/FileUtils.java    |    2 +
 .../org/apache/hadoop/hive/conf/Constants.java  |    4 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    8 +-
 .../apache/hadoop/hive/conf/HiveConfUtil.java   |   91 ++
 .../hive/common/util/HiveStringUtils.java       |   32 +
 data/conf/spark/yarn-client/hive-site.xml       |    4 +-
 data/files/identity_udf.jar                     |  Bin 0 -> 710 bytes
 .../hcatalog/pig/AbstractHCatStorerTest.java    | 1096 ++++++++++++++++++
 .../hive/hcatalog/pig/TestAvroHCatStorer.java   |   77 ++
 .../hive/hcatalog/pig/TestHCatStorer.java       | 1036 ++---------------
 .../hive/hcatalog/pig/TestOrcHCatStorer.java    |   33 +
 .../hcatalog/pig/TestParquetHCatStorer.java     |  200 ++++
 .../hive/hcatalog/pig/TestRCFileHCatStorer.java |   32 +
 .../pig/TestSequenceFileHCatStorer.java         |   33 +
 .../hcatalog/pig/TestTextFileHCatStorer.java    |   33 +
 .../hive/minikdc/TestJdbcWithMiniKdcCookie.java |    1 -
 .../hive/beeline/TestBeeLineWithArgs.java       |   15 +
 .../TestBeelineConnectionUsingHiveSite.java     |  109 ++
 .../TestBeelineWithHS2ConnectionFile.java       |  214 ++++
 .../TestBeelineWithUserHs2ConnectionFile.java   |  129 +++
 metastore/if/hive_metastore.thrift              |    2 +
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |   44 +
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   20 +-
 .../hadoop/hive/metastore/api/TxnInfo.java      |  206 +++-
 .../src/gen/thrift/gen-php/metastore/Types.php  |   46 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   28 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |    6 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |   12 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  675 +++++++----
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |    8 +
 .../hadoop/hive/ql/exec/mr/ExecDriver.java      |    2 +
 .../ql/exec/spark/HiveSparkClientFactory.java   |   14 +
 .../ql/exec/spark/LocalHiveSparkClient.java     |    6 +
 .../ql/exec/spark/RemoteHiveSparkClient.java    |    4 +
 .../rules/HiveSortLimitPullUpConstantsRule.java |   11 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |    6 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   37 +-
 .../hadoop/hive/ql/plan/ShowTxnsDesc.java       |    2 +-
 .../hadoop/hive/ql/util/DependencyResolver.java |    2 +-
 .../ql/exec/TestHiveCredentialProviders.java    |  314 +++++
 .../clientpositive/distinct_windowing_no_cbo.q  |   63 +
 ql/src/test/queries/clientpositive/windowing.q  |    6 +
 .../clientpositive/dbtxnmgr_showlocks.q.out     |    2 +-
 .../distinct_windowing_no_cbo.q.out             |  796 +++++++++++++
 .../llap/cbo_rp_windowing_2.q.out               |    5 +-
 .../results/clientpositive/llap/windowing.q.out |  110 +-
 .../spark/constprog_semijoin.q.out              |   20 +-
 .../clientpositive/spark/index_bitmap3.q.out    |    4 +-
 .../spark/index_bitmap_auto.q.out               |    4 +-
 .../spark/infer_bucket_sort_map_operators.q.out |    8 +-
 .../infer_bucket_sort_reducers_power_two.q.out  |    2 +-
 .../clientpositive/spark/windowing.q.out        |  105 +-
 .../service/cli/operation/OperationManager.java |    3 +
 .../service/cli/thrift/ThriftHttpServlet.java   |    4 +-
 .../cli/session/TestSessionManagerMetrics.java  |   83 +-
 .../thrift/ThriftCliServiceTestWithCookie.java  |    1 -
 .../apache/hadoop/hive/shims/Hadoop23Shims.java |    8 +-
 .../hive/spark/client/SparkClientImpl.java      |   18 +-
 .../ptest2/conf/deployed/master-mr2.properties  |  160 +++
 .../hive/ptest/execution/ExecutionPhase.java    |   21 +-
 .../ptest/execution/conf/QFileTestBatch.java    |   11 +-
 .../ptest/execution/TestExecutionPhase.java     |    8 +-
 .../TestScripts.testPrepGit.approved.txt        |    4 +-
 .../TestScripts.testPrepHadoop1.approved.txt    |    4 +-
 .../TestScripts.testPrepNone.approved.txt       |    4 +-
 .../TestScripts.testPrepSvn.approved.txt        |    4 +-
 83 files changed, 5976 insertions(+), 1303 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index a2997b0,9e5fd37..690cdff
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@@ -1323,171 -1391,180 +1396,216 @@@ public class Driver implements CommandP
      errorMessage = null;
      SQLState = null;
      downstreamError = null;
- 
-     HiveDriverRunHookContext hookContext = new 
HiveDriverRunHookContextImpl(conf,
-         alreadyCompiled ? ctx.getCmd() : command);
-     // Get all the driver run hooks and pre-execute them.
-     List<HiveDriverRunHook> driverRunHooks;
+     stateLock.lock();
      try {
-       driverRunHooks = getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS,
-           HiveDriverRunHook.class);
-       for (HiveDriverRunHook driverRunHook : driverRunHooks) {
-           driverRunHook.preDriverRun(hookContext);
+       if (alreadyCompiled) {
+         if (driverState == DriverState.COMPILED) {
+           driverState = DriverState.EXECUTING;
+         } else {
+           errorMessage = "FAILED: Precompiled query has been cancelled or 
closed.";
+           console.printError(errorMessage);
+           return createProcessorResponse(12);
+         }
+       } else {
+         driverState = DriverState.COMPILING;
        }
-     } catch (Exception e) {
-       errorMessage = "FAILED: Hive Internal Error: " + 
Utilities.getNameMessage(e);
-       SQLState = ErrorMsg.findSQLState(e.getMessage());
-       downstreamError = e;
-       console.printError(errorMessage + "\n"
-           + org.apache.hadoop.util.StringUtils.stringifyException(e));
-       return createProcessorResponse(12);
+     } finally {
+       stateLock.unlock();
      }
  
-     PerfLogger perfLogger = null;
- 
-     int ret;
-     if (!alreadyCompiled) {
-       // compile internal will automatically reset the perf logger
-       ret = compileInternal(command);
-       // then we continue to use this perf logger
-       perfLogger = SessionState.getPerfLogger();
-       if (ret != 0) {
-         return createProcessorResponse(ret);
+     // a flag that helps to set the correct driver state in finally block by 
tracking if
+     // the method has been returned by an error or not.
+     boolean isFinishedWithError = true;
+     try {
+       HiveDriverRunHookContext hookContext = new 
HiveDriverRunHookContextImpl(conf,
+           alreadyCompiled ? ctx.getCmd() : command);
+       // Get all the driver run hooks and pre-execute them.
+       List<HiveDriverRunHook> driverRunHooks;
+       try {
+         driverRunHooks = getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS,
+             HiveDriverRunHook.class);
+         for (HiveDriverRunHook driverRunHook : driverRunHooks) {
+             driverRunHook.preDriverRun(hookContext);
+         }
+       } catch (Exception e) {
+         errorMessage = "FAILED: Hive Internal Error: " + 
Utilities.getNameMessage(e);
+         SQLState = ErrorMsg.findSQLState(e.getMessage());
+         downstreamError = e;
+         console.printError(errorMessage + "\n"
+             + org.apache.hadoop.util.StringUtils.stringifyException(e));
+         return createProcessorResponse(12);
        }
-     } else {
-       // reuse existing perf logger.
-       perfLogger = SessionState.getPerfLogger();
-       // Since we're reusing the compiled plan, we need to update its start 
time for current run
-       plan.setQueryStartTime(perfLogger.getStartTime(PerfLogger.DRIVER_RUN));
-     }
-     // the reason that we set the txn manager for the cxt here is because each
-     // query has its own ctx object. The txn mgr is shared across the
-     // same instance of Driver, which can run multiple queries.
-     HiveTxnManager txnManager = SessionState.get().getTxnMgr();
-     ctx.setHiveTxnManager(txnManager);
- 
-     boolean startTxnImplicitly = false;
-     {
-       //this block ensures op makes sense in given context, e.g. COMMIT is 
valid only if txn is open
-       //DDL is not allowed in a txn, etc.
-       //an error in an open txn does a rollback of the txn
-       if (txnManager.isTxnOpen() && 
!plan.getOperation().isAllowedInTransaction()) {
-         assert !txnManager.getAutoCommit() : "didn't expect AC=true";
-         return rollback(new CommandProcessorResponse(12, 
ErrorMsg.OP_NOT_ALLOWED_IN_TXN, null,
-           plan.getOperationName(), 
Long.toString(txnManager.getCurrentTxnId())));
-       }
-       if(!txnManager.isTxnOpen() && 
plan.getOperation().isRequiresOpenTransaction()) {
-         return rollback(new CommandProcessorResponse(12, 
ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN, null, plan.getOperationName()));
-       }
-       if(!txnManager.isTxnOpen() && plan.getOperation() == 
HiveOperation.QUERY && !txnManager.getAutoCommit()) {
-         //this effectively makes START TRANSACTION optional and supports JDBC 
setAutoCommit(false) semantics
-         //also, indirectly allows DDL to be executed outside a txn context
-         startTxnImplicitly = true;
-       }
-       if(txnManager.getAutoCommit() && plan.getOperation() == 
HiveOperation.START_TRANSACTION) {
+ 
+       PerfLogger perfLogger = null;
+ 
+       int ret;
+       if (!alreadyCompiled) {
+         // compile internal will automatically reset the perf logger
+         ret = compileInternal(command, true);
+         // then we continue to use this perf logger
+         perfLogger = SessionState.getPerfLogger();
+         if (ret != 0) {
+           return createProcessorResponse(ret);
+         }
+       } else {
+         // reuse existing perf logger.
+         perfLogger = SessionState.getPerfLogger();
+         // Since we're reusing the compiled plan, we need to update its start 
time for current run
+         
plan.setQueryStartTime(perfLogger.getStartTime(PerfLogger.DRIVER_RUN));
+       }
+       // the reason that we set the txn manager for the cxt here is because 
each
+       // query has its own ctx object. The txn mgr is shared across the
+       // same instance of Driver, which can run multiple queries.
+       HiveTxnManager txnManager = SessionState.get().getTxnMgr();
+       ctx.setHiveTxnManager(txnManager);
+ 
+       boolean startTxnImplicitly = false;
+       {
+         //this block ensures op makes sense in given context, e.g. COMMIT is 
valid only if txn is open
+         //DDL is not allowed in a txn, etc.
+         //an error in an open txn does a rollback of the txn
+         if (txnManager.isTxnOpen() && 
!plan.getOperation().isAllowedInTransaction()) {
+           assert !txnManager.getAutoCommit() : "didn't expect AC=true";
+           return rollback(new CommandProcessorResponse(12, 
ErrorMsg.OP_NOT_ALLOWED_IN_TXN, null,
+             plan.getOperationName(), 
Long.toString(txnManager.getCurrentTxnId())));
+         }
+         if(!txnManager.isTxnOpen() && 
plan.getOperation().isRequiresOpenTransaction()) {
+           return rollback(new CommandProcessorResponse(12, 
ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN, null, plan.getOperationName()));
+         }
+         if(!txnManager.isTxnOpen() && plan.getOperation() == 
HiveOperation.QUERY && !txnManager.getAutoCommit()) {
+           //this effectively makes START TRANSACTION optional and supports 
JDBC setAutoCommit(false) semantics
+           //also, indirectly allows DDL to be executed outside a txn context
+           startTxnImplicitly = true;
+         }
+         if(txnManager.getAutoCommit() && plan.getOperation() == 
HiveOperation.START_TRANSACTION) {
            return rollback(new CommandProcessorResponse(12, 
ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT, null, plan.getOperationName()));
+         }
        }
-     }
-     if(plan.getOperation() == HiveOperation.SET_AUTOCOMMIT) {
-       try {
-         if(plan.getAutoCommitValue() && !txnManager.getAutoCommit()) {
-           /*here, if there is an open txn, we want to commit it; this 
behavior matches
-           * 
https://docs.oracle.com/javase/6/docs/api/java/sql/Connection.html#setAutoCommit(boolean)*/
-           releaseLocksAndCommitOrRollback(true, null);
-           txnManager.setAutoCommit(true);
+       if(plan.getOperation() == HiveOperation.SET_AUTOCOMMIT) {
+         try {
+           if(plan.getAutoCommitValue() && !txnManager.getAutoCommit()) {
+             /*here, if there is an open txn, we want to commit it; this 
behavior matches
+             * 
https://docs.oracle.com/javase/6/docs/api/java/sql/Connection.html#setAutoCommit(boolean)*/
+             releaseLocksAndCommitOrRollback(true, null);
+             txnManager.setAutoCommit(true);
+           }
+           else if(!plan.getAutoCommitValue() && txnManager.getAutoCommit()) {
+             txnManager.setAutoCommit(false);
+           }
+           else {/*didn't change autoCommit value - no-op*/}
          }
-         else if(!plan.getAutoCommitValue() && txnManager.getAutoCommit()) {
-           txnManager.setAutoCommit(false);
+         catch(LockException e) {
+           return handleHiveException(e, 12);
          }
-         else {/*didn't change autoCommit value - no-op*/}
        }
-       catch(LockException e) {
-         return handleHiveException(e, 12);
+ 
+       if (requiresLock()) {
+         // a checkpoint to see if the thread is interrupted or not before an 
expensive operation
+         if (isInterrupted()) {
+           ret = handleInterruption("at acquiring the lock.");
+         } else {
+           ret = acquireLocksAndOpenTxn(startTxnImplicitly);
+         }
+         if (ret != 0) {
+           return rollback(createProcessorResponse(ret));
+         }
+       }
++
++      try {
++        acquireWriteIds(plan, conf);
++      } catch (HiveException e) {
++        return handleHiveException(e, 1);
 +      }
-     }
 +
-     if (requiresLock()) {
-       ret = acquireLocksAndOpenTxn(startTxnImplicitly);
+       ret = execute(true);
        if (ret != 0) {
+         //if needRequireLock is false, the release here will do nothing 
because there is no lock
          return rollback(createProcessorResponse(ret));
        }
-     }
-     try {
-       acquireWriteIds(plan, conf);
-     } catch (HiveException e) {
-       return handleHiveException(e, 1);
-     }
-     ret = execute();
-     if (ret != 0) {
-       //if needRequireLock is false, the release here will do nothing because 
there is no lock
-       return rollback(createProcessorResponse(ret));
-     }
  
-     //if needRequireLock is false, the release here will do nothing because 
there is no lock
-     try {
-       if(txnManager.getAutoCommit() || plan.getOperation() == 
HiveOperation.COMMIT) {
-         releaseLocksAndCommitOrRollback(true, null);
-       }
-       else if(plan.getOperation() == HiveOperation.ROLLBACK) {
-         releaseLocksAndCommitOrRollback(false, null);
-       }
-       else {
-         //txn (if there is one started) is not finished
+       //if needRequireLock is false, the release here will do nothing because 
there is no lock
+       try {
+         if(txnManager.getAutoCommit() || plan.getOperation() == 
HiveOperation.COMMIT) {
+           releaseLocksAndCommitOrRollback(true, null);
+         }
+         else if(plan.getOperation() == HiveOperation.ROLLBACK) {
+           releaseLocksAndCommitOrRollback(false, null);
+         }
+         else {
+           //txn (if there is one started) is not finished
+         }
+       } catch (LockException e) {
+         return handleHiveException(e, 12);
        }
-     } catch (LockException e) {
-       return handleHiveException(e, 12);
-     }
  
-     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN);
-     queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, 
perfLogger.getStartTimes());
-     queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, 
perfLogger.getEndTimes());
+       perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN);
+       queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, 
perfLogger.getStartTimes());
+       queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, 
perfLogger.getEndTimes());
  
-     // Take all the driver run hooks and post-execute them.
-     try {
-       for (HiveDriverRunHook driverRunHook : driverRunHooks) {
-           driverRunHook.postDriverRun(hookContext);
+       // Take all the driver run hooks and post-execute them.
+       try {
+         for (HiveDriverRunHook driverRunHook : driverRunHooks) {
+             driverRunHook.postDriverRun(hookContext);
+         }
+       } catch (Exception e) {
+         errorMessage = "FAILED: Hive Internal Error: " + 
Utilities.getNameMessage(e);
+         SQLState = ErrorMsg.findSQLState(e.getMessage());
+         downstreamError = e;
+         console.printError(errorMessage + "\n"
+             + org.apache.hadoop.util.StringUtils.stringifyException(e));
+         return createProcessorResponse(12);
+       }
+       isFinishedWithError = false;
+       return createProcessorResponse(ret);
+     } finally {
+       if (isInterrupted()) {
+         closeInProcess(true);
+       } else {
+         // only release the related resources ctx, driverContext as normal
+         releaseResources();
+       }
+       stateLock.lock();
+       try {
+         if (driverState == DriverState.INTERRUPT) {
+           driverState = DriverState.ERROR;
+         } else {
+           driverState = isFinishedWithError ? DriverState.ERROR : 
DriverState.EXECUTED;
+         }
+       } finally {
+         stateLock.unlock();
        }
-     } catch (Exception e) {
-       errorMessage = "FAILED: Hive Internal Error: " + 
Utilities.getNameMessage(e);
-       SQLState = ErrorMsg.findSQLState(e.getMessage());
-       downstreamError = e;
-       console.printError(errorMessage + "\n"
-           + org.apache.hadoop.util.StringUtils.stringifyException(e));
-       return createProcessorResponse(12);
      }
- 
-     return createProcessorResponse(ret);
    }
  
 +  private static void acquireWriteIds(QueryPlan plan, HiveConf conf) throws 
HiveException {
 +    // Output IDs are put directly into FileSinkDesc; here, we only need to 
take care of inputs.
 +    for (ReadEntity input : plan.getInputs()) {
 +      Table t = extractMmTable(input);
 +      if (t == null) continue;
 +      ValidWriteIds ids = Hive.get().getValidWriteIdsForTable(t.getDbName(), 
t.getTableName());
 +      ids.addToConf(conf, t.getDbName(), t.getTableName());
 +      if (plan.getFetchTask() != null) {
 +        ids.addToConf(plan.getFetchTask().getFetchConf(), t.getDbName(), 
t.getTableName());
 +      }
 +    }
 +  }
 +
 +  private static Table extractMmTable(ReadEntity input) {
 +    Table t = null;
 +    switch (input.getType()) {
 +      case TABLE:
 +        t = input.getTable();
 +        break;
 +      case DUMMYPARTITION:
 +      case PARTITION:
 +        t = input.getPartition().getTable();
 +        break;
 +      default: return null;
 +    }
 +    return (t != null && !t.isTemporary()
 +        && MetaStoreUtils.isMmTable(t.getParameters())) ? t : null;
 +  }
 +
    private CommandProcessorResponse rollback(CommandProcessorResponse cpr) {
      //console.printError(cpr.toString());
      try {

http://git-wip-us.apache.org/repos/asf/hive/blob/2474f063/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

Reply via email to