This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4fa5a3920b6 HIVE-28615: OTEL: Expose Tez counters for queries (#5535)
4fa5a3920b6 is described below
commit 4fa5a3920b652c2ee3f3c81529af0d97364855d4
Author: Tanishq Chugh <[email protected]>
AuthorDate: Fri Mar 13 20:52:51 2026 +0530
HIVE-28615: OTEL: Expose Tez counters for queries (#5535)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 5 +++-
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 4 ++-
.../org/apache/hadoop/hive/ql/DriverUtils.java | 8 ++++++
.../org/apache/hadoop/hive/ql/QueryDisplay.java | 31 ++++++++++++++++++++++
.../apache/hive/service/servlet/OTELExporter.java | 3 +++
5 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 90e9b2b7094..c45c1f533ce 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5759,7 +5759,10 @@ public static enum ConfVars {
HIVE_OTEL_RETRY_BACKOFF_MULTIPLIER("hive.otel.retry.backoff.multiplier",
5f,
"The multiplier applied to the backoff interval for retries in the
OTEL exporter."
- + "This determines how much the backoff interval increases after
each failed attempt, following an exponential backoff strategy.");
+ + "This determines how much the backoff interval increases after
each failed attempt, following an exponential backoff strategy."),
+
+ HIVE_OTEL_EXPORT_TEZ_COUNTERS("hive.otel.export.tez.counters", false,
+ "Enables the inclusion of Tez counters in OTEL output for a particular
query.");
public final String varname;
public final String altName;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 4ad2c82c568..58d4d9354ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -203,7 +203,9 @@ private void runInternal(String command, boolean
alreadyCompiled) throws Command
driverContext.getQueryDisplay().setPerfLogStarts(QueryDisplay.Phase.EXECUTION,
perfLogger.getStartTimes());
driverContext.getQueryDisplay().setPerfLogEnds(QueryDisplay.Phase.EXECUTION,
perfLogger.getEndTimes());
-
+ if (DriverUtils.isOtelExportTezCountersEnabled(driverContext)) {
+
driverContext.getQueryDisplay().setTezCounters(driverContext.getRuntimeContext().getCounters());
+ }
runPostDriverHooks(hookContext);
isFinishedWithError = false;
} finally {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
index ad1ac2d9a6a..ba3dbb478dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -181,4 +182,11 @@ public static String getUserFromUGI(DriverContext
driverContext) throws CommandP
throw createProcessorException(driverContext, 10, errorMessage,
ErrorMsg.findSQLState(e.getMessage()), e);
}
}
+
+ public static boolean isOtelExportTezCountersEnabled(DriverContext
driverContext) {
+ return driverContext.getRuntimeContext() != null &&
+
driverContext.getConf().getTimeVar(HiveConf.ConfVars.HIVE_OTEL_METRICS_FREQUENCY_SECONDS,
+ TimeUnit.MILLISECONDS) > 0 &&
+
driverContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_OTEL_EXPORT_TEZ_COUNTERS);
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
index 35eccd2909a..981328d1257 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
@@ -21,6 +21,9 @@
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskResult;
import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.tez.common.counters.CounterGroup;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.common.counters.TezCounters;
import java.io.IOException;
import java.util.*;
@@ -37,6 +40,7 @@
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* Some limited query information to save for WebUI.
@@ -58,6 +62,7 @@ public class QueryDisplay {
private String explainPlan;
private String errorMessage;
private String queryId;
+ private TezCounters tezCounters;
private long queryStartTime = System.currentTimeMillis();
private final Map<Phase, Map<String, Long>> hmsTimingMap = new
HashMap<Phase, Map<String, Long>>();
@@ -305,6 +310,32 @@ public synchronized void setExplainPlan(String
explainPlan) {
this.explainPlan = explainPlan;
}
+ public synchronized void setTezCounters(TezCounters tc) {
+ this.tezCounters = tc;
+ }
+
+ public synchronized TezCounters getTezCounters() {
+ return this.tezCounters;
+ }
+
+ public synchronized Map<String, String> getCountersAsString() {
+ Map<String, String> allCounterGroups = new HashMap<>();
+ if (tezCounters != null) {
+ for (CounterGroup group : tezCounters) {
+ ObjectNode groupNode = OBJECT_MAPPER.createObjectNode();
+ for (TezCounter counter : group) {
+ groupNode.put(counter.getDisplayName(), counter.getValue());
+ }
+ String counterGroupName = group.getName();
+ String[] counterGroupSplits = counterGroupName.split("\\.");
+ counterGroupName = counterGroupName.contains(".") ?
counterGroupSplits[counterGroupSplits.length - 1] :
+ counterGroupName + " Counter";
+ allCounterGroups.put(counterGroupName, groupNode.toString());
+ }
+ }
+ return allCounterGroups;
+ }
+
/**
* @param phase phase of query
* @return map of HMS Client method-calls and duration in milliseconds,
during given phase.
diff --git a/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
index 1fb53715ed0..7a592c12521 100644
--- a/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
+++ b/service/src/java/org/apache/hive/service/servlet/OTELExporter.java
@@ -214,6 +214,9 @@ private AttributesMap addQueryAttributes(QueryInfo query){
attributes.put(AttributeKey.stringKey("UserName"), query.getUserName());
attributes.put(AttributeKey.stringKey("State"), query.getState());
attributes.put(AttributeKey.stringKey("SessionId"), query.getSessionId());
+ for (Map.Entry<String, String> entry :
query.getQueryDisplay().getCountersAsString().entrySet()) {
+ attributes.put(AttributeKey.stringKey(entry.getKey()), entry.getValue());
+ }
return attributes;
}
