This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b44f5c3cd4802cd215f51dcd7698370a8e45ea78 Author: Michael Blow <[email protected]> AuthorDate: Thu Jul 9 17:01:03 2020 -0400 [NO ISSUE] Redact sensitive data from statement logging Change-Id: Ibd63ca9167c769eea4d03982dbf7fa543913dc67 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/7165 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- .../api/http/server/QueryServiceServlet.java | 7 ++-- .../message/ExecuteStatementRequestMessage.java | 2 +- .../asterix/hyracks/bootstrap/CCApplication.java | 3 ++ .../asterix/hyracks/bootstrap/NCApplication.java | 3 ++ .../org/apache/asterix/utils/RedactionUtil.java | 41 ++++++++++++---------- .../java/org/apache/hyracks/util/ILogRedactor.java | 8 +++++ .../org/apache/hyracks/util/LogRedactionUtil.java | 9 +++++ 7 files changed, 50 insertions(+), 23 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index cb1d6cf..440b351 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -279,7 +279,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { if (forceReadOnly) { param.setReadOnly(true); } - LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.userData(param.toString())); + LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString())); delivery = param.getMode(); setSessionConfig(sessionOutput, param, delivery); final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads()); @@ -427,10 +427,11 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { if (t instanceof org.apache.asterix.aqlplus.parser.TokenMgrError || t instanceof TokenMgrError || t instanceof AlgebricksException) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.userData(param.toString()), t); + LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()), + t); } else { LOGGER.info(() -> "handleException: " + t.getMessage() + ": " - + LogRedactionUtil.userData(param.toString())); + + LogRedactionUtil.statement(param.toString())); } executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); } else if (t instanceof HyracksException) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java index 149ed33..2eced12 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java @@ -204,6 +204,6 @@ public final class ExecuteStatementRequestMessage implements ICcAddressedMessage @Override public String toString() { return String.format("%s(id=%s, from=%s): %s", getClass().getSimpleName(), requestMessageId, requestNodeId, - LogRedactionUtil.userData(statementsText)); + LogRedactionUtil.statement(statementsText)); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java index fc912b0..0f0620c 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java @@ -86,6 +86,7 @@ import org.apache.asterix.runtime.utils.CcApplicationContext; import org.apache.asterix.translator.IStatementExecutorFactory; import org.apache.asterix.translator.Receptionist; import org.apache.asterix.util.MetadataBuiltinFunctions; +import org.apache.asterix.utils.RedactionUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.client.IHyracksClientConnection; @@ -104,6 +105,7 @@ import org.apache.hyracks.http.server.HttpServerConfig; import org.apache.hyracks.http.server.HttpServerConfigBuilder; import org.apache.hyracks.http.server.WebManager; import org.apache.hyracks.ipc.impl.HyracksConnection; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.LoggingConfigUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -204,6 +206,7 @@ public class CCApplication extends BaseCCApplication { public void configureLoggingLevel(Level level) { super.configureLoggingLevel(level); LoggingConfigUtil.defaultIfMissing(GlobalConfig.ASTERIX_LOGGER_NAME, level); + LogRedactionUtil.setRedactor(RedactionUtil.LOG_REDACTOR); } protected List<AsterixExtension> getExtensions() throws Exception { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 2e5c09c..1036fb2 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -72,6 +72,7 @@ import org.apache.asterix.messaging.NCMessageBroker; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository; import org.apache.asterix.translator.Receptionist; import org.apache.asterix.util.MetadataBuiltinFunctions; +import org.apache.asterix.utils.RedactionUtil; import org.apache.hyracks.api.application.IServiceContext; import org.apache.hyracks.api.client.NodeStatus; import org.apache.hyracks.api.config.IConfigManager; @@ -86,6 +87,7 @@ import org.apache.hyracks.http.server.HttpServer; import org.apache.hyracks.http.server.HttpServerConfig; import org.apache.hyracks.http.server.HttpServerConfigBuilder; import org.apache.hyracks.http.server.WebManager; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.LoggingConfigUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -184,6 +186,7 @@ public class NCApplication extends BaseNCApplication { public void configureLoggingLevel(Level level) { super.configureLoggingLevel(level); LoggingConfigUtil.defaultIfMissing(GlobalConfig.ASTERIX_LOGGER_NAME, level); + LogRedactionUtil.setRedactor(RedactionUtil.LOG_REDACTOR); } protected void configureServers() throws Exception { diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java similarity index 53% copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java index 89c957e..156b78a 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java @@ -16,36 +16,39 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.asterix.utils; -package org.apache.hyracks.util; +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static java.util.regex.Pattern.DOTALL; +import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME; -public class LogRedactionUtil { +import java.util.regex.Pattern; - private static final ILogRedactor DEFAULT_LOG_REDACTOR = new ILogRedactor() { +import org.apache.hyracks.util.ILogRedactor; + +public class RedactionUtil { + private RedactionUtil() { + throw new AssertionError("do not instantiate"); + } + + private static final Pattern STATEMENT_PATTERN = + Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL); + private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>"; + + public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() { @Override public String userData(String text) { return text; } @Override + public String statement(String text) { + return STATEMENT_PATTERN.matcher(text).replaceFirst(STATEMENT_REPLACEMENT); + } + + @Override public String unredactUserData(String text) { return text; } }; - private static ILogRedactor redactor = DEFAULT_LOG_REDACTOR; - - private LogRedactionUtil() { - } - - public static void setRedactor(ILogRedactor redactor) { - LogRedactionUtil.redactor = redactor; - } - - public static String userData(String text) { - return redactor.userData(text); - } - - public static String unredactUserData(String text) { - return redactor.unredactUserData(text); - } } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java index b133894..d36c77b 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java @@ -29,6 +29,14 @@ public interface ILogRedactor { String userData(String text); /** + * Redacts statement argument. + * + * @param text statement to redact. + * @return redacted statement. + */ + String statement(String text); + + /** * Unredacts user data found in the argument. * * @param text text that contains some redacted user data. diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java index 89c957e..fdce8c8 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java @@ -28,6 +28,11 @@ public class LogRedactionUtil { } @Override + public String statement(String text) { + return text; + } + + @Override public String unredactUserData(String text) { return text; } @@ -45,6 +50,10 @@ public class LogRedactionUtil { return redactor.userData(text); } + public static String statement(String text) { + return redactor.statement(text); + } + public static String unredactUserData(String text) { return redactor.unredactUserData(text); }
