This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a91d6af17c6 Conditionally Log MSE stats when query finish. (#16007)
a91d6af17c6 is described below
commit a91d6af17c651f139a4fdcc0e090de3c91eb8b8a
Author: Gonzalo Ortiz Jaureguizar <[email protected]>
AuthorDate: Wed Jun 18 08:52:51 2025 +0200
Conditionally Log MSE stats when query finish. (#16007)
---
.../MultiStageBrokerRequestHandler.java | 41 +++++++++++++++++++++-
.../apache/pinot/spi/utils/CommonConstants.java | 8 +++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
index e1b5f040271..374c9c1e8b9 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
@@ -20,12 +20,14 @@ package org.apache.pinot.broker.requesthandler;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -93,6 +95,8 @@ import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.sql.parsers.SqlNodeAndOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
+import org.slf4j.MarkerFactory;
/**
@@ -101,6 +105,19 @@ import org.slf4j.LoggerFactory;
*/
public class MultiStageBrokerRequestHandler extends BaseBrokerRequestHandler {
private static final Logger LOGGER =
LoggerFactory.getLogger(MultiStageBrokerRequestHandler.class);
+ /// Disabled by default, but can be enabled with
+ ///```xml
+ /// <MarkerFilter marker="MSE_STATS_MARKER" onMatch="ACCEPT"
onMismatch="NEUTRAL"/>
+ /// ...
+ /// <Loggers>
+ /// <Logger name="org.apache.pinot" level="debug" additivity="false">
+ /// <AppenderRef ref="console">
+ /// <MarkerFilter marker="MSE_STATS_MARKER"/>
+ /// </AppenderRef>
+ /// </Logger>
+ /// </Loggers>
+ /// ```
+ private static final Marker MSE_STATS_MARKER =
MarkerFactory.getMarker("MSE_STATS_MARKER");
private static final int NUM_UNAVAILABLE_SEGMENTS_TO_LOG = 10;
@@ -170,9 +187,9 @@ public class MultiStageBrokerRequestHandler extends
BaseBrokerRequestHandler {
requestContext, httpHeaders);
if (!brokerResponse.getExceptions().isEmpty()) {
// a _green_ error (see handleRequestThrowing javadoc)
- LOGGER.info("Request {} failed in a controlled manner: {}", requestId,
brokerResponse.getExceptions());
onFailedRequest(brokerResponse.getExceptions());
}
+ summarizeQuery(brokerResponse,
explicitSummarizeLogRequested(sqlNodeAndOptions));
return brokerResponse;
} catch (WebApplicationException e) {
// a _yellow_ error (see handleRequestThrowing javadoc)
@@ -200,6 +217,28 @@ public class MultiStageBrokerRequestHandler extends
BaseBrokerRequestHandler {
}
}
+ private static boolean explicitSummarizeLogRequested(SqlNodeAndOptions
sqlNodeAndOptions) {
+ return Boolean.parseBoolean(
+ sqlNodeAndOptions.getOptions()
+
.getOrDefault(CommonConstants.MultiStageQueryRunner.KEY_OF_LOG_STATS, "false")
+ .toLowerCase(Locale.US));
+ }
+
+ private void summarizeQuery(BrokerResponse brokerResponse, boolean
explicitSummarizeLogRequested) {
+ ObjectNode stats = brokerResponse instanceof BrokerResponseNativeV2
+ ? ((BrokerResponseNativeV2) brokerResponse).getStageStats()
+ : JsonNodeFactory.instance.objectNode();
+ String completionStatus = brokerResponse.getExceptions().isEmpty()
+ ? "successfully"
+ : "with errors " + brokerResponse.getExceptions();
+ String logTemplate = "Request finished {} in {}ms. Stats: {}";
+ if (brokerResponse.getExceptions().isEmpty() &&
!explicitSummarizeLogRequested) {
+ LOGGER.debug(MSE_STATS_MARKER, logTemplate, completionStatus,
brokerResponse.getTimeUsedMs(), stats);
+ } else {
+ LOGGER.info(MSE_STATS_MARKER, logTemplate, completionStatus,
brokerResponse.getTimeUsedMs(), stats);
+ }
+ }
+
private void onFailedRequest(List<QueryProcessingException> exs) {
_brokerMetrics.addMeteredGlobalValue(BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS,
1);
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 962179f999b..4c3c4876df6 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1661,6 +1661,14 @@ public class CommonConstants {
public static final String KEY_OF_SEND_STATS_MODE =
"pinot.query.mse.stats.mode";
public static final String DEFAULT_SEND_STATS_MODE = "SAFE";
+ /// Used to indicate that MSE stats should be logged at INFO level for
successful queries.
+ ///
+ /// When an MSE query is executed, the stats are collected and logged.
+ /// By default, successful queries are logged in the DEBUG level, while
errors are logged in the INFO level.
+ /// But if this property is set to true (upper or lower case), stats will
be logged in the INFO level for both
+ /// successful queries and errors.
+ public static final String KEY_OF_LOG_STATS = "logStats";
+
public enum JoinOverFlowMode {
THROW, BREAK
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
