This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 265f183 HIVE-23118: Option for exposing compile time counters as tez
counters (Prasanth Jayachandran reviewed by Jesus Camacho Rodriguez)
265f183 is described below
commit 265f183338c786e47e72d9617dfef3c3023b7a4f
Author: Prasanth Jayachandran <[email protected]>
AuthorDate: Fri Apr 3 18:08:22 2020 -0700
HIVE-23118: Option for exposing compile time counters as tez counters
(Prasanth Jayachandran reviewed by Jesus Camacho Rodriguez)
Signed-off-by: Prasanth Jayachandran <[email protected]>
---
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 +++++++--
.../org/apache/hadoop/hive/ql/exec/tez/Utils.java | 26 ++++++++++++++++++++++
.../hive/ql/exec/tez/monitoring/TezJobMonitor.java | 12 +++++++---
.../hive/ql/hooks/PostExecTezSummaryPrinter.java | 6 +++++
5 files changed, 50 insertions(+), 5 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 3e89071..f54c96e 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -689,6 +689,7 @@ minillaplocal.query.files=\
orc_llap.q,\
orc_llap_nonvector.q,\
orc_ppd_date.q,\
+ tez_compile_counters.q,\
tez_input_counters.q,\
orc_ppd_decimal.q,\
orc_ppd_timestamp.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 0bacb05..3599d19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -127,6 +127,9 @@ public class TezTask extends Task<TezWork> {
return counters;
}
+ public void setTezCounters(final TezCounters counters) {
+ this.counters = counters;
+ }
@Override
public int execute() {
@@ -235,7 +238,7 @@ public class TezTask extends Task<TezWork> {
}
// finally monitor will print progress until the job is done
- TezJobMonitor monitor = new TezJobMonitor(work.getAllWork(),
dagClient, conf, dag, ctx);
+ TezJobMonitor monitor = new TezJobMonitor(work.getAllWork(),
dagClient, conf, dag, ctx, counters);
rc = monitor.monitorExecution();
if (rc != 0) {
@@ -245,7 +248,10 @@ public class TezTask extends Task<TezWork> {
// fetch the counters
try {
Set<StatusGetOpts> statusGetOpts =
EnumSet.of(StatusGetOpts.GET_COUNTERS);
- counters = dagClient.getDAGStatus(statusGetOpts).getDAGCounters();
+ TezCounters dagCounters =
dagClient.getDAGStatus(statusGetOpts).getDAGCounters();
+ // if initial counters exists, merge it with dag counters to get
aggregated view
+ TezCounters mergedCounters = counters == null ? dagCounters :
Utils.mergeTezCounters(dagCounters, counters);
+ counters = mergedCounters;
} catch (Exception err) {
// Don't fail execution due to counters - just don't print summary
info
LOG.warn("Failed to get counters. Ignoring, summary info will be
incomplete. " + err, err);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
index cc9c4ce..d266bb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
@@ -30,6 +30,7 @@ import
org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.split.SplitLocationProvider;
+import org.apache.tez.common.counters.TezCounters;
import org.slf4j.Logger;
public class Utils {
@@ -101,4 +102,29 @@ public class Utils {
}
return new HostAffinitySplitLocationProvider(locations);
}
+
+
+ /**
+ * Merges two different tez counters into one
+ *
+ * @param counter1 - tez counter 1
+ * @param counter2 - tez counter 2
+ * @return - merged tez counter
+ */
+ public static TezCounters mergeTezCounters(final TezCounters counter1, final
TezCounters counter2) {
+ TezCounters merged = new TezCounters();
+ if (counter1 != null) {
+ for (String counterGroup : counter1.getGroupNames()) {
+ merged.addGroup(counter1.getGroup(counterGroup));
+ }
+ }
+
+ if (counter2 != null) {
+ for (String counterGroup : counter2.getGroupNames()) {
+ merged.addGroup(counter2.getGroup(counterGroup));
+ }
+ }
+
+ return merged;
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
index d3fe190..9729a7b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
+import org.apache.hadoop.hive.ql.exec.tez.Utils;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -112,9 +113,11 @@ public class TezJobMonitor {
private final Context context;
private long executionStartTime = 0;
private final RenderStrategy.UpdateFunction updateFunction;
+ // compile time tez counters
+ private final TezCounters counters;
public TezJobMonitor(List<BaseWork> topSortedWorks, final DAGClient
dagClient, HiveConf conf, DAG dag,
- Context ctx) {
+ Context ctx, final TezCounters counters) {
this.topSortedWorks = topSortedWorks;
this.dagClient = dagClient;
this.hiveConf = conf;
@@ -122,6 +125,7 @@ public class TezJobMonitor {
this.context = ctx;
console = SessionState.getConsole();
updateFunction = updateFunction();
+ this.counters = counters;
}
private RenderStrategy.UpdateFunction updateFunction() {
@@ -185,8 +189,10 @@ public class TezJobMonitor {
if (wmContext != null) {
Set<String> desiredCounters = wmContext.getSubscribedCounters();
TezCounters dagCounters = status.getDAGCounters();
- if (dagCounters != null && desiredCounters != null &&
!desiredCounters.isEmpty()) {
- Map<String, Long> currentCounters = getCounterValues(dagCounters,
vertexNames, vertexProgressMap,
+ // if initial counters exists, merge it with dag counters to get
aggregated view
+ TezCounters mergedCounters = counters == null ? dagCounters :
Utils.mergeTezCounters(dagCounters, counters);
+ if (mergedCounters != null && desiredCounters != null &&
!desiredCounters.isEmpty()) {
+ Map<String, Long> currentCounters =
getCounterValues(mergedCounters, vertexNames, vertexProgressMap,
desiredCounters, done);
if (LOG.isDebugEnabled()) {
LOG.debug("Requested DAG status. checkInterval: {}.
currentCounters: {}", checkInterval, currentCounters);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
index 14ebfa0..8b5ed93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.hooks;
import java.util.List;
import org.apache.hadoop.hive.llap.counters.LlapIOCounters;
+import org.apache.hadoop.hive.ql.exec.tez.CompileTimeCounters;
import org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters;
import org.apache.tez.common.counters.FileSystemCounter;
import org.apache.tez.dag.api.client.DAGClient;
@@ -90,6 +91,11 @@ public class PostExecTezSummaryPrinter implements
ExecuteWithHookContext {
console.printInfo(" " + counter.getDisplayName() + ": " +
counter.getValue(), false);
}
}
+ } else if
(group.getName().equals(CompileTimeCounters.class.getName())) {
+ console.printInfo(tezTask.getId() + " COMPILE TIME COUNTERS:",
false);
+ for (TezCounter counter : group) {
+ console.printInfo(" " + counter.getDisplayName() + ": " +
counter.getValue(), false);
+ }
}
}
}
