This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9079019423f96566997d933ef789058c1e28baba Author: Ali Alsuliman <[email protected]> AuthorDate: Fri Dec 8 09:00:59 2023 -0800 [NO ISSUE][API] Add query request param to indicate source - user model changes: no - storage format changes: no - interface changes: no Details: Add parameter 'source' to QueryServiceRequestParameters to indicate the source of the request. Change-Id: I04e2e2336d6bca82b4fa44a43cb188cfa9e807f7 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- .../http/server/QueryServiceRequestParameters.java | 22 +++++++++++++++++++--- .../api/http/server/QueryServiceServlet.java | 20 +++++++++----------- .../message/ExecuteStatementRequestMessage.java | 5 +++-- .../asterix/app/translator/QueryTranslator.java | 6 +++--- .../org/apache/asterix/utils/RedactionUtil.java | 3 +++ 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java index b6913e448b..20d79d5e7e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java @@ -19,6 +19,8 @@ package org.apache.asterix.api.http.server; +import static org.apache.asterix.utils.RedactionUtil.REDACTED_SENSITIVE_ENTRY_VALUE; + import java.io.IOException; import java.util.HashMap; import java.util.Iterator; @@ -43,6 +45,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.server.utils.HttpUtil; import org.apache.hyracks.util.JSONUtil; +import org.apache.hyracks.util.LogRedactionUtil; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; @@ -80,7 +83,8 @@ public class QueryServiceRequestParameters { SIGNATURE("signature"), MULTI_STATEMENT("multi-statement"), MAX_WARNINGS("max-warnings"), - SQL_COMPAT("sql-compat"); + SQL_COMPAT("sql-compat"), + SOURCE("source"); private final String str; @@ -124,6 +128,7 @@ public class QueryServiceRequestParameters { private String statement; private String clientContextID; private String dataverse; + private String source; private ClientType clientType = ClientType.ASTERIX; private OutputFormat format = OutputFormat.CLEAN_JSON; private ResultDelivery mode = ResultDelivery.IMMEDIATE; @@ -172,6 +177,14 @@ public class QueryServiceRequestParameters { this.statement = statement; } + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + public OutputFormat getFormat() { return format; } @@ -380,7 +393,8 @@ public class QueryServiceRequestParameters { ObjectNode object = OBJECT_MAPPER.createObjectNode(); object.put("host", host); object.put("path", path); - object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null); + object.put("statement", statement != null + ? LogRedactionUtil.statement(JSONUtil.escape(new StringBuilder(), statement).toString()) : null); object.put("pretty", pretty); object.put("mode", mode.getName()); object.put("clientContextID", clientContextID); @@ -402,9 +416,10 @@ public class QueryServiceRequestParameters { object.put("readOnly", readOnly); object.put("maxWarnings", maxWarnings); object.put("sqlCompat", sqlCompatMode); + object.put("source", source); if (statementParams != null) { for (Map.Entry<String, JsonNode> statementParam : statementParams.entrySet()) { - object.set('$' + statementParam.getKey(), statementParam.getValue()); + object.set('$' + statementParam.getKey(), REDACTED_SENSITIVE_ENTRY_VALUE); } } return object; @@ -486,6 +501,7 @@ public class QueryServiceRequestParameters { setSignature(parseBoolean(req, Parameter.SIGNATURE.str(), valGetter, isSignature())); setClientType(parseIfExists(req, Parameter.CLIENT_TYPE.str(), valGetter, getClientType(), clientTypes::get)); setSQLCompatMode(parseBoolean(req, Parameter.SQL_COMPAT.str(), valGetter, isSQLCompatMode())); + setSource(valGetter.apply(req, Parameter.SOURCE.str())); } protected void setExtraParams(JsonNode jsonRequest) throws HyracksDataException { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 1966a88ba6..9a8c0d5b23 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -90,7 +90,6 @@ import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; import org.apache.hyracks.http.server.utils.HttpUtil; -import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -278,7 +277,10 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { if (forceReadOnly) { param.setReadOnly(true); } - LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("handleRequest: uuid={}, clientContextID={}, {}", requestRef.getUuid(), + param.getClientContextID(), param.toString()); + } delivery = param.getMode(); setSessionConfig(sessionOutput, param, delivery); final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads()); @@ -429,14 +431,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); return true; case REQUEST_TIMEOUT: - LOGGER.info(() -> "handleException: request execution timed out: " - + LogRedactionUtil.userData(param.toString())); + LOGGER.info(() -> "handleException: request execution timed out: " + param.toString()); executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK); return true; case REJECT_NODE_UNREGISTERED: case REJECT_BAD_CLUSTER_STATE: - LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " - + LogRedactionUtil.userData(param.toString())); + LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " + param.toString()); executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE); return true; default: @@ -456,11 +456,9 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { QueryServiceRequestParameters param, IServletResponse response) { if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()), - t); + LOGGER.debug("handleException: {}: {}", t.getMessage(), param.toString(), t); } else { - LOGGER.info(() -> "handleException: " + t.getMessage() + ": " - + LogRedactionUtil.statement(param.toString())); + LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + param.toString()); } executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); return; @@ -472,7 +470,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { return; } } - LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), t); + LOGGER.warn(() -> "handleException: unexpected exception: " + param.toString(), t); executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java index 05bc87b2f5..e31417798b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java @@ -243,7 +243,8 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage { @Override public String toString() { - return String.format("%s(id=%s, from=%s, uuid=%s): %s", getClass().getSimpleName(), requestMessageId, - requestNodeId, requestReference.getUuid(), LogRedactionUtil.statement(statementsText)); + return String.format("%s(id=%s, from=%s, uuid=%s, clientContextID=%s): %s", getClass().getSimpleName(), + requestMessageId, requestNodeId, requestReference.getUuid(), clientContextID, + LogRedactionUtil.statement(statementsText)); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index afdd0359a4..6b40cbfffd 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -4837,9 +4837,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen // ensure request not cancelled before running job ensureNotCancelled(clientRequest); final JobId jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("createAndRunJob jobId:{}, uuid:{}", jobId, - requestParameters.getRequestReference().getUuid()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Created job {} for query uuid:{}, clientContextID:{}", jobId, + requestParameters.getRequestReference().getUuid(), requestParameters.getClientContextId()); } clientRequest.setJobId(jobId); if (jId != null) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java index 156b78ae93..d8356643a3 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java @@ -26,6 +26,8 @@ import java.util.regex.Pattern; import org.apache.hyracks.util.ILogRedactor; +import com.fasterxml.jackson.databind.node.TextNode; + public class RedactionUtil { private RedactionUtil() { throw new AssertionError("do not instantiate"); @@ -34,6 +36,7 @@ public class RedactionUtil { private static final Pattern STATEMENT_PATTERN = Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL); private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>"; + public static final TextNode REDACTED_SENSITIVE_ENTRY_VALUE = new TextNode("<redacted sensitive entry>"); public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() { @Override
