This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 928f3d6 HIVE-21591: Using triggers in non-LLAP mode should not
require wm queue (Prasanth Jayachandran reviewed by Daniel Dai)
928f3d6 is described below
commit 928f3d65c324c785565cdcc1d809695b1a5344bd
Author: Prasanth Jayachandran <prasan...@apache.org>
AuthorDate: Tue Apr 9 15:01:19 2019 -0700
HIVE-21591: Using triggers in non-LLAP mode should not require wm queue
(Prasanth Jayachandran reviewed by Daniel Dai)
---
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 5 +--
.../hive/ql/exec/tez/TezSessionPoolManager.java | 15 ++++++---
.../apache/hive/service/server/HiveServer2.java | 37 +++++++++++-----------
3 files changed, 33 insertions(+), 24 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 269cd85..a1d795f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -26,6 +26,7 @@ import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
@@ -582,8 +583,8 @@ public class DDLTask extends Task<DDLWork> implements
Serializable {
}
}
if (pm != null) {
- pm.updateTriggers(appliedRp);
- LOG.info("Updated tez session pool manager with active resource plan:
{}", name);
+ Collection<String> appliedTriggers = pm.updateTriggers(appliedRp);
+ LOG.info("Updated tez session pool manager with active resource plan: {}
appliedTriggers: {}", name, appliedTriggers);
}
return 0;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index 2633390..7c0a1fe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
@@ -21,9 +21,12 @@ package org.apache.hadoop.hive.ql.exec.tez;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionState.HiveResources;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -113,9 +116,9 @@ public class TezSessionPoolManager extends
TezSessionPoolSession.AbstractTrigger
}
initTriggers(conf);
if (resourcePlan != null) {
- updateTriggers(resourcePlan);
- LOG.info("Updated tez session pool manager with active resource plan:
{}",
- resourcePlan.getPlan().getName());
+ Collection<String> appliedTriggers = updateTriggers(resourcePlan);
+ LOG.info("Updated tez session pool manager with triggers {} from active
resource plan: {}",
+ appliedTriggers, resourcePlan.getPlan().getName());
}
}
@@ -531,7 +534,8 @@ public class TezSessionPoolManager extends
TezSessionPoolSession.AbstractTrigger
}
}
- public void updateTriggers(final WMFullResourcePlan appliedRp) {
+ public Collection<String> updateTriggers(final WMFullResourcePlan appliedRp)
{
+ Set<String> triggerNames = new HashSet<>();
if (sessionTriggerProvider != null) {
List<WMTrigger> wmTriggers = appliedRp != null ? appliedRp.getTriggers()
: null;
List<Trigger> triggers = new ArrayList<>();
@@ -539,11 +543,14 @@ public class TezSessionPoolManager extends
TezSessionPoolSession.AbstractTrigger
for (WMTrigger wmTrigger : wmTriggers) {
if (wmTrigger.isSetIsInUnmanaged() && wmTrigger.isIsInUnmanaged()) {
triggers.add(ExecutionTrigger.fromWMTrigger(wmTrigger));
+ triggerNames.add(wmTrigger.getTriggerName());
}
}
}
sessionTriggerProvider.setTriggers(Collections.unmodifiableList(triggers));
}
+
+ return triggerNames;
}
/** Called by TezSessionPoolSession when closed. */
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java
b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 452b8d8..00f6e90 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -732,23 +732,21 @@ public class HiveServer2 extends CompositeService {
private void startOrReconnectTezSessions() {
LOG.info("Starting/Reconnecting tez sessions..");
// TODO: add tez session reconnect after TEZ-3875
- WMFullResourcePlan resourcePlan = null;
- if (!StringUtils.isEmpty(wmQueue)) {
- try {
- resourcePlan = sessionHive.getActiveResourcePlan();
- } catch (HiveException e) {
- if (!HiveConf.getBoolVar(getHiveConf(), ConfVars.HIVE_IN_TEST_SSL)) {
- throw new RuntimeException(e);
- } else {
- resourcePlan = null; // Ignore errors in SSL tests where the
connection is misconfigured.
- }
+ WMFullResourcePlan resourcePlan;
+ try {
+ resourcePlan = sessionHive.getActiveResourcePlan();
+ } catch (HiveException e) {
+ if (!HiveConf.getBoolVar(getHiveConf(), ConfVars.HIVE_IN_TEST_SSL)) {
+ throw new RuntimeException(e);
+ } else {
+ resourcePlan = null; // Ignore errors in SSL tests where the
connection is misconfigured.
}
+ }
- if (resourcePlan == null && HiveConf.getBoolVar(
- getHiveConf(), ConfVars.HIVE_IN_TEST)) {
- LOG.info("Creating a default resource plan for test");
- resourcePlan = createTestResourcePlan();
- }
+ if (resourcePlan == null && HiveConf.getBoolVar(
+ getHiveConf(), ConfVars.HIVE_IN_TEST)) {
+ LOG.info("Creating a default resource plan for test");
+ resourcePlan = createTestResourcePlan();
}
initAndStartTezSessionPoolManager(resourcePlan);
initAndStartWorkloadManager(resourcePlan);
@@ -763,7 +761,8 @@ public class HiveServer2 extends CompositeService {
// SessionState.get() return null during createTezDir
try {
// will be invoked anyway in TezTask. Doing it early to initialize
triggers for non-pool tez session.
- LOG.info("Initializing tez session pool manager");
+ LOG.info("Initializing tez session pool manager. Active resource plan:
{}",
+ resourcePlan == null || resourcePlan.getPlan() == null ? "null" :
resourcePlan.getPlan().getName());
tezSessionPoolManager = TezSessionPoolManager.getInstance();
HiveConf hiveConf = getHiveConf();
if
(hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS)) {
@@ -781,7 +780,8 @@ public class HiveServer2 extends CompositeService {
private void initAndStartWorkloadManager(final WMFullResourcePlan
resourcePlan) {
if (!StringUtils.isEmpty(wmQueue)) {
// Initialize workload management.
- LOG.info("Initializing workload management");
+ LOG.info("Initializing workload management. Active resource plan: {}",
+ resourcePlan == null || resourcePlan.getPlan() == null ? "null" :
resourcePlan.getPlan().getName());
try {
wm = WorkloadManager.create(wmQueue, getHiveConf(), resourcePlan);
wm.start();
@@ -790,7 +790,8 @@ public class HiveServer2 extends CompositeService {
throw new ServiceException("Unable to instantiate and start Workload
Manager", e);
}
} else {
- LOG.info("Workload management is not enabled.");
+ LOG.info("Workload management is not enabled as {} config is not set",
+ ConfVars.HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE.varname);
}
}
