This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
commit 27a15244a7d7d8ba7c09f580d2b862ca9419057f Author: Zoltan Haindrich <k...@rxd.hu> AuthorDate: Thu Nov 28 14:01:22 2019 +0000 HIVE-22521: Both Driver and SessionState has a userName (Zoltan Haindrich reviewed by László Bodor) Signed-off-by: Zoltan Haindrich <zhaindr...@cloudera.com> --- .../ql/exec/spark/TestSmallTableCacheEviction.java | 2 +- .../ql/exec/spark/TestSparkSessionTimeout.java | 2 +- .../hive/ql/exec/spark/TestSparkStatistics.java | 2 +- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 26 +++++++++------------- .../org/apache/hadoop/hive/ql/DriverContext.java | 8 +------ .../org/apache/hadoop/hive/ql/DriverFactory.java | 8 +++---- .../org/apache/hadoop/hive/ql/DriverUtils.java | 2 +- .../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 8 +++---- .../apache/hadoop/hive/ql/reexec/ReExecDriver.java | 5 ++--- .../scheduled/ScheduledQueryExecutionService.java | 2 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 2 +- .../hive/ql/udf/generic/GenericUDTFGetSplits.java | 2 +- .../org/apache/hadoop/hive/ql/TestTxnCommands.java | 2 +- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 2 +- .../apache/hadoop/hive/ql/TestTxnCommands3.java | 2 +- .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 2 +- .../ql/exec/spark/TestLocalHiveSparkClient.java | 2 +- .../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 4 ++-- .../hive/service/cli/operation/SQLOperation.java | 2 +- 19 files changed, 36 insertions(+), 49 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java index 7bdb5e5..bf7f3dc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java @@ -104,7 +104,7 @@ public class TestSmallTableCacheEviction { return new Driver(new QueryState.Builder() .withGenerateNewQueryId(true) .withHiveConf(conf).build(), - null, null); + null); } /** diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java index 51bd8a7..842aa9f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java @@ -111,7 +111,7 @@ public class TestSparkSessionTimeout { driver = new Driver(new QueryState.Builder() .withGenerateNewQueryId(true) .withHiveConf(conf).build(), - null, null); + null); SparkSession sparkSession = SparkUtilities.getSparkSession(conf, SparkSessionManagerImpl .getInstance()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java index 2b72d09..9b433a2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java @@ -58,7 +58,7 @@ public class TestSparkStatistics { driver = new Driver(new QueryState.Builder() .withGenerateNewQueryId(true) .withHiveConf(conf).build(), - null, null); + null); driver.run("create table test (col int)"); Assert.assertEquals(0, driver.compile("select * from test order by col", true)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 1574982..a28bf16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -235,32 +235,26 @@ public class Driver implements IDriver { @VisibleForTesting public Driver(HiveConf conf) { - this(new QueryState.Builder().withGenerateNewQueryId(true).withHiveConf(conf).build(), null); + this(new QueryState.Builder().withGenerateNewQueryId(true).withHiveConf(conf).build()); } // Pass lineageState when a driver instantiates another Driver to run // or compile another query public Driver(HiveConf conf, Context ctx, LineageState lineageState) { - this(getNewQueryState(conf, lineageState), null, null); + this(getNewQueryState(conf, lineageState), null); this.ctx = ctx; } - // Pass lineageState when a driver instantiates another Driver to run - // or compile another query - public Driver(HiveConf conf, String userName, LineageState lineageState) { - this(getNewQueryState(conf, lineageState), userName, null); - } - - public Driver(QueryState queryState, String userName) { - this(queryState, userName, null, null); + public Driver(QueryState queryState) { + this(queryState, null, null); } - public Driver(QueryState queryState, String userName, QueryInfo queryInfo) { - this(queryState, userName, queryInfo, null); + public Driver(QueryState queryState, QueryInfo queryInfo) { + this(queryState, queryInfo, null); } - public Driver(QueryState queryState, String userName, QueryInfo queryInfo, HiveTxnManager txnManager) { - driverContext = new DriverContext(queryState, queryInfo, userName, new HookRunner(queryState.getConf(), CONSOLE), + public Driver(QueryState queryState, QueryInfo queryInfo, HiveTxnManager txnManager) { + driverContext = new DriverContext(queryState, queryInfo, new HookRunner(queryState.getConf(), CONSOLE), txnManager); } @@ -270,7 +264,7 @@ public class Driver implements IDriver { * @param lineageState a LineageState to be set in the new QueryState object * @return The new QueryState object */ - private static QueryState getNewQueryState(HiveConf conf, LineageState lineageState) { + public static QueryState getNewQueryState(HiveConf conf, LineageState lineageState) { return new QueryState.Builder() .withGenerateNewQueryId(true) .withHiveConf(conf) @@ -409,7 +403,7 @@ public class Driver implements IDriver { HiveSemanticAnalyzerHookContext hookCtx = new HiveSemanticAnalyzerHookContextImpl(); if (executeHooks) { hookCtx.setConf(driverContext.getConf()); - hookCtx.setUserName(driverContext.getUserName()); + hookCtx.setUserName(SessionState.get().getUserName()); hookCtx.setIpAddress(SessionState.get().getUserIpAddress()); hookCtx.setCommand(command); hookCtx.setHiveOperation(driverContext.getQueryState().getHiveOperation()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java index 1afcfc8..13e2f29 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java @@ -38,7 +38,6 @@ public class DriverContext { private final QueryState queryState; private final QueryInfo queryInfo; private final HiveConf conf; - private final String userName; private final HookRunner hookRunner; // Transaction manager the Driver has been initialized with (can be null). @@ -72,12 +71,11 @@ public class DriverContext { private Context backupContext = null; private boolean retrial = false; - public DriverContext(QueryState queryState, QueryInfo queryInfo, String userName, HookRunner hookRunner, + public DriverContext(QueryState queryState, QueryInfo queryInfo, HookRunner hookRunner, HiveTxnManager initTxnManager) { this.queryState = queryState; this.queryInfo = queryInfo; this.conf = queryState.getConf(); - this.userName = userName; this.hookRunner = hookRunner; this.initTxnManager = initTxnManager; } @@ -98,10 +96,6 @@ public class DriverContext { return conf; } - public String getUserName() { - return userName; - } - public HookRunner getHookRunner() { return hookRunner; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java index 963741c..9cca2a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverFactory.java @@ -35,13 +35,13 @@ import com.google.common.base.Strings; public class DriverFactory { public static IDriver newDriver(HiveConf conf) { - return newDriver(getNewQueryState(conf), null, null); + return newDriver(getNewQueryState(conf), null); } - public static IDriver newDriver(QueryState queryState, String userName, QueryInfo queryInfo) { + public static IDriver newDriver(QueryState queryState, QueryInfo queryInfo) { boolean enabled = queryState.getConf().getBoolVar(ConfVars.HIVE_QUERY_REEXECUTION_ENABLED); if (!enabled) { - return new Driver(queryState, userName, queryInfo); + return new Driver(queryState, queryInfo); } String strategies = queryState.getConf().getVar(ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES); @@ -54,7 +54,7 @@ public class DriverFactory { plugins.add(buildReExecPlugin(string)); } - return new ReExecDriver(queryState, userName, queryInfo, plugins); + return new ReExecDriver(queryState, queryInfo, plugins); } private static IReExecutionPlugin buildReExecPlugin(String name) throws RuntimeException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java index 26e904a..aa8e64d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java @@ -48,7 +48,7 @@ public class DriverUtils { boolean isOk = false; try { QueryState qs = new QueryState.Builder().withHiveConf(conf).withGenerateNewQueryId(true).nonIsolated().build(); - Driver driver = new Driver(qs, user, null, null); + Driver driver = new Driver(qs, null, null); driver.setCompactionWriteIds(writeIds, compactorTxnId); try { try { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 5ebf719..497b17f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -2341,10 +2341,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { } cmd.append(")"); } - SessionState ss = SessionState.get(); - // TODO: should this use getUserFromAuthenticator? - String uName = (ss == null? null: ss.getUserName()); - Driver driver = new Driver(conf, uName, queryState.getLineageState()); + // FIXME: is it ok to have a completely new querystate? + QueryState newQueryState = Driver.getNewQueryState(conf, queryState.getLineageState()); + // FIXME: this driver instance is never closed + Driver driver = new Driver(newQueryState); int rc = driver.compile(cmd.toString(), false); if (rc != 0) { throw new SemanticException(ErrorMsg.NO_VALID_PARTN.getMsg()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java index dea46c0..a32af75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReExecDriver.java @@ -103,10 +103,9 @@ public class ReExecDriver implements IDriver { return executionIndex == 0; } - public ReExecDriver(QueryState queryState, String userName, QueryInfo queryInfo, - ArrayList<IReExecutionPlugin> plugins) { + public ReExecDriver(QueryState queryState, QueryInfo queryInfo, ArrayList<IReExecutionPlugin> plugins) { this.queryState = queryState; - coreDriver = new Driver(queryState, userName, queryInfo, null); + coreDriver = new Driver(queryState, queryInfo, null); coreDriver.getHookRunner().addSemanticAnalyzerHook(new HandleReOptimizationExplain()); this.plugins = plugins; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java b/ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java index 5dd105f..48bdc97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/scheduled/ScheduledQueryExecutionService.java @@ -118,7 +118,7 @@ public class ScheduledQueryExecutionService implements Closeable { info.setState(QueryState.EXECUTING); reportQueryProgress(); try ( - IDriver driver = DriverFactory.newDriver(DriverFactory.getNewQueryState(conf), q.getUser(), null)) { + IDriver driver = DriverFactory.newDriver(DriverFactory.getNewQueryState(conf), null)) { info.setExecutorQueryId(driver.getQueryState().getQueryId()); driver.run(q.getQuery()); info.setState(QueryState.FINISHED); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java index 3270175..749cdb6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java @@ -342,7 +342,7 @@ public class Worker extends RemoteCompactorThread implements MetaStoreThread { //todo: use DriverUtils.runOnDriver() here QueryState queryState = new QueryState.Builder().withGenerateNewQueryId(true).withHiveConf(conf).build(); SessionState localSession = null; - try (Driver d = new Driver(queryState, userName)) { + try (Driver d = new Driver(queryState)) { if (SessionState.get() == null) { localSession = new SessionState(conf); SessionState.start(localSession); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java index e95edbf..389f5cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java @@ -306,7 +306,7 @@ public class GenericUDTFGetSplits extends GenericUDTF { // So initialize the new Driver with a new TxnManager so that it does not use the // Session TxnManager that is already in use. HiveTxnManager txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf); - Driver driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null, null, txnManager); + Driver driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null, txnManager); DriverCleanup driverCleanup = new DriverCleanup(driver, txnManager, splitsAppId.toString()); boolean needsCleanup = true; try { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 6e7b201..8421408 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -272,7 +272,7 @@ public class TestTxnCommands extends TxnCommandsBaseForTests { throw new RuntimeException(e); } QueryState qs = new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build(); - try (Driver d = new Driver(qs, null)) { + try (Driver d = new Driver(qs)) { LOG.info("Ready to run the query: " + query); syncThreadStart(cdlIn, cdlOut); try { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index cbc72b4..c184ce5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -160,7 +160,7 @@ public class TestTxnCommands2 { } SessionState ss = SessionState.start(hiveConf); ss.applyAuthorizationPolicy(); - d = new Driver(new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build(), null); + d = new Driver(new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build()); d.setMaxRows(10000); dropTables(); runStatementOnDriver("create table " + Table.ACIDTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES (" + tableProperties + ")"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java index 7b3ab28..908ceb4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java @@ -306,7 +306,7 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests { runStatementOnDriver("insert into T values(0,2)");//makes delta_1_1 in T1 runStatementOnDriver("insert into T values(1,4)");//makes delta_2_2 in T2 - Driver driver2 = new Driver(new QueryState.Builder().withHiveConf(hiveConf).build(), null); + Driver driver2 = new Driver(new QueryState.Builder().withHiveConf(hiveConf).build()); driver2.setMaxRows(10000); HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index f01a07e..af14e62 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -114,7 +114,7 @@ public abstract class TxnCommandsBaseForTests { } SessionState ss = SessionState.start(hiveConf); ss.applyAuthorizationPolicy(); - d = new Driver(new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build(), null); + d = new Driver(new QueryState.Builder().withHiveConf(hiveConf).nonIsolated().build()); d.setMaxRows(10000); dropTables(); runStatementOnDriver("create table " + Table.ACIDTBL + "(a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')"); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestLocalHiveSparkClient.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestLocalHiveSparkClient.java index 94991d3..3f89b90 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestLocalHiveSparkClient.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestLocalHiveSparkClient.java @@ -91,7 +91,7 @@ public class TestLocalHiveSparkClient { try { driver = new Driver(new QueryState.Builder() .withGenerateNewQueryId(true) - .withHiveConf(conf).build(), null, null); + .withHiveConf(conf).build(), null); SparkSession sparkSession = SparkUtilities.getSparkSession(conf, SparkSessionManagerImpl.getInstance()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index f9bbb28..23d860f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -100,8 +100,8 @@ public class TestDbTxnManager2 { public void setUp() throws Exception { SessionState.start(conf); ctx = new Context(conf); - driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null); - driver2 = new Driver(new QueryState.Builder().withHiveConf(conf).build(), null); + driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build()); + driver2 = new Driver(new QueryState.Builder().withHiveConf(conf).build()); TxnDbUtil.cleanDb(conf); TxnDbUtil.prepDb(conf); SessionState ss = SessionState.get(); diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index b87b670..468ce10 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -160,7 +160,7 @@ public class SQLOperation extends ExecuteStatementOperation { public void prepare(QueryState queryState) throws HiveSQLException { setState(OperationState.RUNNING); try { - driver = DriverFactory.newDriver(queryState, getParentSession().getUserName(), queryInfo); + driver = DriverFactory.newDriver(queryState, queryInfo); // Start the timer thread for cancelling the query when query timeout is reached // queryTimeout == 0 means no timeout