This is an automated email from the ASF dual-hosted git repository. vy pushed a commit to branch LOG4J2-2993 in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit 78e70a7e4c2ccf4d5a7a96317488c6b2f6bf61c5 Author: Volkan Yazici <[email protected]> AuthorDate: Fri Jan 15 16:43:57 2021 +0100 LOG4J2-2993 Support stack trace truncation in JsonTemplateLayout. --- log4j-layout-template-json/revapi.json | 26 +++ .../layout/template/json/JsonTemplateLayout.java | 5 +- .../json/resolver/EventResolverContext.java | 18 ++ .../template/json/resolver/ExceptionResolver.java | 192 +++++++++++++++++++-- .../json/resolver/ExceptionRootCauseResolver.java | 2 +- .../json/resolver/StackTraceStringResolver.java | 80 ++++++++- .../layout/template/json/util/MapAccessor.java | 51 +++++- .../json/util/TruncatingBufferedPrintWriter.java | 50 +++++- .../json/util/TruncatingBufferedWriter.java | 69 +++++++- .../src/main/resources/EcsLayout.json | 4 +- .../src/main/resources/GelfLayout.json | 4 +- .../main/resources/LogstashJsonEventLayoutV1.json | 4 +- .../template/json/JsonTemplateLayoutTest.java | 130 +++++++++++++- .../json/util/TruncatingBufferedWriterTest.java | 20 +-- .../src/test/resources/testJsonTemplateLayout.json | 4 +- src/changes/changes.xml | 3 + .../asciidoc/manual/json-template-layout.adoc.vm | 70 ++++++-- 17 files changed, 664 insertions(+), 68 deletions(-) diff --git a/log4j-layout-template-json/revapi.json b/log4j-layout-template-json/revapi.json index a259b85..77f8cde 100644 --- a/log4j-layout-template-json/revapi.json +++ b/log4j-layout-template-json/revapi.json @@ -408,6 +408,32 @@ "old": "enum org.apache.logging.log4j.layout.template.json.util.Uris", "new": "class org.apache.logging.log4j.layout.template.json.util.Uris", "justification": "Replaced 'enum' singletons with 'final class'es." + }, + { + "code": "java.method.removed", + "old": "method char[] org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedPrintWriter::getBuffer()", + "justification": "LOG4J2-2993 Massaged (internal) API to make method names more Java-like and restrict access if possible." + }, + { + "code": "java.method.removed", + "old": "method int org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedPrintWriter::getCapacity()", + "justification": "LOG4J2-2993 Massaged (internal) API to make method names more Java-like and restrict access if possible." + }, + { + "code": "java.method.removed", + "old": "method int org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedPrintWriter::getPosition()", + "justification": "LOG4J2-2993 Massaged (internal) API to make method names more Java-like and restrict access if possible." + }, + { + "code": "java.method.removed", + "old": "method boolean org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedPrintWriter::isTruncated()", + "justification": "LOG4J2-2993 Massaged (internal) API to make method names more Java-like and restrict access if possible." + }, + { + "code": "java.class.visibilityReduced", + "old": "class org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedWriter", + "new": "class org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedWriter", + "justification": "LOG4J2-2993 Massaged (internal) API to make method names more Java-like and restrict access if possible." } ] } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayout.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayout.java index 1f0506d..1ef2b13 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayout.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayout.java @@ -137,8 +137,8 @@ public class JsonTemplateLayout implements StringLayout { final String eventTemplate = readEventTemplate(builder); final float maxByteCountPerChar = builder.charset.newEncoder().maxBytesPerChar(); final int maxStringByteCount = - Math.toIntExact(Math.round( - maxByteCountPerChar * builder.maxStringLength)); + Math.toIntExact(Math.round(Math.ceil( + maxByteCountPerChar * builder.maxStringLength))); final EventTemplateAdditionalField[] eventTemplateAdditionalFields = builder.eventTemplateAdditionalFields != null ? builder.eventTemplateAdditionalFields @@ -151,6 +151,7 @@ public class JsonTemplateLayout implements StringLayout { .setJsonWriter(jsonWriter) .setRecyclerFactory(builder.recyclerFactory) .setMaxStringByteCount(maxStringByteCount) + .setTruncatedStringSuffix(builder.truncatedStringSuffix) .setLocationInfoEnabled(builder.locationInfoEnabled) .setStackTraceEnabled(builder.stackTraceEnabled) .setStackTraceElementObjectResolver(stackTraceElementObjectResolver) diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/EventResolverContext.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/EventResolverContext.java index 8f7107b..e1d2cb6 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/EventResolverContext.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/EventResolverContext.java @@ -41,6 +41,8 @@ public final class EventResolverContext implements TemplateResolverContext<LogEv private final int maxStringByteCount; + private final String truncatedStringSuffix; + private final boolean locationInfoEnabled; private final boolean stackTraceEnabled; @@ -58,6 +60,7 @@ public final class EventResolverContext implements TemplateResolverContext<LogEv this.jsonWriter = builder.jsonWriter; this.recyclerFactory = builder.recyclerFactory; this.maxStringByteCount = builder.maxStringByteCount; + this.truncatedStringSuffix = builder.truncatedStringSuffix; this.locationInfoEnabled = builder.locationInfoEnabled; this.stackTraceEnabled = builder.stackTraceEnabled; this.stackTraceObjectResolver = stackTraceEnabled @@ -103,6 +106,10 @@ public final class EventResolverContext implements TemplateResolverContext<LogEv return maxStringByteCount; } + String getTruncatedStringSuffix() { + return truncatedStringSuffix; + } + boolean isLocationInfoEnabled() { return locationInfoEnabled; } @@ -141,6 +148,8 @@ public final class EventResolverContext implements TemplateResolverContext<LogEv private int maxStringByteCount; + private String truncatedStringSuffix; + private boolean locationInfoEnabled; private boolean stackTraceEnabled; @@ -185,6 +194,15 @@ public final class EventResolverContext implements TemplateResolverContext<LogEv return this; } + public String getTruncatedStringSuffix() { + return truncatedStringSuffix; + } + + public Builder setTruncatedStringSuffix(final String truncatedStringSuffix) { + this.truncatedStringSuffix = truncatedStringSuffix; + return this; + } + public Builder setLocationInfoEnabled(final boolean locationInfoEnabled) { this.locationInfoEnabled = locationInfoEnabled; return this; diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionResolver.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionResolver.java index 5fc7a6d..424f1d4 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionResolver.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionResolver.java @@ -17,18 +17,100 @@ package org.apache.logging.log4j.layout.template.json.resolver; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.layout.template.json.JsonTemplateLayout; +import org.apache.logging.log4j.layout.template.json.JsonTemplateLayoutDefaults; import org.apache.logging.log4j.layout.template.json.util.JsonWriter; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + /** * Exception resolver. * * <h3>Configuration</h3> * * <pre> - * config = field , [ stringified ] - * field = "field" -> ( "className" | "message" | "stackTrace" ) - * stringified = "stringified" -> boolean + * config = field , [ stringified ] , [ stackTrace ] + * field = "field" -> ( "className" | "message" | "stackTrace" ) + * stackTrace = "stackTrace" -> ( + * [ stringified ] + * , [ truncatedStringSuffix ] + * , [ truncationPointMatcherStrings ] + * , [ truncationPointMatcherRegexes ] + * ) + * stringified = "stringified" -> boolean + * truncatedStringSuffix = "truncatedStringSuffix" -> string + * truncationPointMatcherStrings = "truncationPointMatcherStrings" -> string[] + * truncationPointMatcherRegexes = "truncationPointMatcherRegexes" -> string[] + * </pre> + * + * <tt>stringified</tt> is set to <tt>false</tt> by default. + * <tt>stringified</tt> at the root level is <b>deprecated</b>, + * instead prefer the one in the <tt>stackTrace</tt> object, which has + * precedence if both are provided. + * <p> + * <tt>truncationPointMatcherStrings</tt> and + * <tt>truncationPointMatcherRegexes</tt> enable the truncation of the stack + * trace after the given matching point. If both parameters are provided, + * <tt>truncationPointMatcherStrings</tt> will be checked first. Note that + * these configurations are only taken into account when <tt>stringified</tt> + * is set to <tt>true</tt>. + * <p> + * <tt>truncatedStringSuffix</tt> will be set to the one configured in the + * layout, unless explicitly provided. + * + * <h3>Examples</h3> + * + * Resolve <tt>logEvent.getThrown().getClass().getCanonicalName()</tt>: + * + * <pre> + * { + * "$resolver": "exception", + * "field": "className" + * } + * </pre> + * + * Resolve the stack trace into a list of <tt>StackTraceElement</tt> objects: + * + * <pre> + * { + * "$resolver": "exception", + * "field": "stackTrace" + * } + * </pre> + * + * Resolve the stack trace into a string field: + * + * <pre> + * { + * "$resolver": "exception", + * "field": "stackTrace", + * "stackTrace": { + * "stringified": true + * } + * } + * </pre> + * + * Resolve the stack trace into a string field + * such that the content will be truncated by the given points: + * + * <pre> + * { + * "$resolver": "exception", + * "field": "stackTrace", + * "stackTrace": { + * "stringified": true, + * "truncatedStringSuffix": ">", + * "truncationPointStrings": ["at javax.servlet.http.HttpServlet.service"] + * } + * } * </pre> + * + * @see JsonTemplateLayout.Builder#getTruncatedStringSuffix() + * @see JsonTemplateLayoutDefaults#getTruncatedStringSuffix() + * @see ExceptionRootCauseResolver */ class ExceptionResolver implements EventResolver { @@ -89,37 +171,123 @@ class ExceptionResolver implements EventResolver { if (!context.isStackTraceEnabled()) { return NULL_RESOLVER; } - final boolean stringified = config.getBoolean("stringified", false); + final boolean stringified = isStackTraceStringified(config); return stringified - ? createStackTraceStringResolver(context) + ? createStackTraceStringResolver(context, config) : createStackTraceObjectResolver(context); } - private EventResolver createStackTraceStringResolver(EventResolverContext context) { - StackTraceStringResolver stackTraceStringResolver = - new StackTraceStringResolver(context); + private static boolean isStackTraceStringified( + final TemplateResolverConfig config) { + final Boolean stringifiedOld = config.getBoolean("stringified"); + final Boolean stringifiedNew = + config.getBoolean(new String[]{"stackTrace", "stringified"}); + if (stringifiedOld == null && stringifiedNew == null) { + return false; + } else if (stringifiedNew == null) { + return stringifiedOld; + } else { + return stringifiedNew; + } + } + + private EventResolver createStackTraceStringResolver( + final EventResolverContext context, + final TemplateResolverConfig config) { + + // Read the configuration. + final String truncatedStringSuffix = + readTruncatedStringSuffix(context, config); + final List<String> truncationPointMatcherStrings = + readTruncationPointMatcherStrings(config); + final List<String> truncationPointMatcherRegexes = + readTruncationPointMatcherRegexes(config); + + // Create the resolver. + final StackTraceStringResolver resolver = + new StackTraceStringResolver( + context, + truncatedStringSuffix, + truncationPointMatcherStrings, + truncationPointMatcherRegexes); + + // Create the null-protected resolver. return (final LogEvent logEvent, final JsonWriter jsonWriter) -> { final Throwable exception = extractThrowable(logEvent); if (exception == null) { jsonWriter.writeNull(); } else { - stackTraceStringResolver.resolve(exception, jsonWriter); + resolver.resolve(exception, jsonWriter); } }; + + } + + private static String readTruncatedStringSuffix( + final EventResolverContext context, + final TemplateResolverConfig config) { + final String suffix = config.getString( + new String[]{"stackTrace", "truncatedStringSuffix"}); + return suffix != null + ? suffix + : context.getTruncatedStringSuffix(); + } + + private static List<String> readTruncationPointMatcherStrings( + final TemplateResolverConfig config) { + List<String> strings = config.getList( + new String[]{"stackTrace", "truncationPointMatcherStrings"}, + String.class); + if (strings == null) { + strings = Collections.emptyList(); + } + return strings; + } + + private static List<String> readTruncationPointMatcherRegexes( + final TemplateResolverConfig config) { + + // Extract the regexes. + List<String> regexes = config.getList( + new String[]{"stackTrace", "truncationPointMatcherRegexes"}, + String.class); + if (regexes == null) { + regexes = Collections.emptyList(); + } + + // Check the regex syntax. + for (int i = 0; i < regexes.size(); i++) { + final String regex = regexes.get(i); + try { + Pattern.compile(regex); + } catch (final PatternSyntaxException error) { + final String message = String.format( + "invalid truncation point matcher regex at index %d: %s", + i, regex); + throw new IllegalArgumentException(message, error); + } + } + + // Return the extracted regexes. + return regexes; + } - private EventResolver createStackTraceObjectResolver(EventResolverContext context) { + private EventResolver createStackTraceObjectResolver( + final EventResolverContext context) { return (final LogEvent logEvent, final JsonWriter jsonWriter) -> { final Throwable exception = extractThrowable(logEvent); if (exception == null) { jsonWriter.writeNull(); } else { - context.getStackTraceObjectResolver().resolve(exception, jsonWriter); + context + .getStackTraceObjectResolver() + .resolve(exception, jsonWriter); } }; } - Throwable extractThrowable(LogEvent logEvent) { + Throwable extractThrowable(final LogEvent logEvent) { return logEvent.getThrown(); } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionRootCauseResolver.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionRootCauseResolver.java index dfa9d0a..37119ca 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionRootCauseResolver.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ExceptionRootCauseResolver.java @@ -41,7 +41,7 @@ final class ExceptionRootCauseResolver extends ExceptionResolver { } @Override - Throwable extractThrowable(LogEvent logEvent) { + Throwable extractThrowable(final LogEvent logEvent) { final Throwable thrown = logEvent.getThrown(); return thrown != null ? Throwables.getRootCause(thrown) : null; } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/StackTraceStringResolver.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/StackTraceStringResolver.java index 725ac1e..412c038 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/StackTraceStringResolver.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/StackTraceStringResolver.java @@ -20,19 +20,52 @@ import org.apache.logging.log4j.layout.template.json.util.TruncatingBufferedPrin import org.apache.logging.log4j.layout.template.json.util.JsonWriter; import org.apache.logging.log4j.layout.template.json.util.Recycler; +import java.util.List; import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; final class StackTraceStringResolver implements StackTraceResolver { private final Recycler<TruncatingBufferedPrintWriter> writerRecycler; - StackTraceStringResolver(final EventResolverContext context) { + private final String truncatedStringSuffix; + + private final boolean truncationEnabled; + + private final List<String> truncationPointMatcherStrings; + + private final List<Pattern> groupedTruncationPointMatcherRegexes; + + StackTraceStringResolver( + final EventResolverContext context, + final String truncatedStringSuffix, + final List<String> truncationPointMatcherStrings, + final List<String> truncationPointMatcherRegexes) { final Supplier<TruncatingBufferedPrintWriter> writerSupplier = () -> TruncatingBufferedPrintWriter.ofCapacity( context.getMaxStringByteCount()); this.writerRecycler = context .getRecyclerFactory() .create(writerSupplier, TruncatingBufferedPrintWriter::close); + this.truncationEnabled = + !truncationPointMatcherStrings.isEmpty() || + !truncationPointMatcherRegexes.isEmpty(); + this.truncatedStringSuffix = truncatedStringSuffix; + this.truncationPointMatcherStrings = truncationPointMatcherStrings; + this.groupedTruncationPointMatcherRegexes = + groupTruncationPointMatcherRegexes(truncationPointMatcherRegexes); + } + + private static List<Pattern> groupTruncationPointMatcherRegexes( + final List<String> regexes) { + return regexes + .stream() + .map(regex -> Pattern.compile( + "^.*(" + regex + ")(.*)$", + Pattern.MULTILINE | Pattern.DOTALL)) + .collect(Collectors.toList()); } @Override @@ -42,10 +75,53 @@ final class StackTraceStringResolver implements StackTraceResolver { final TruncatingBufferedPrintWriter writer = writerRecycler.acquire(); try { throwable.printStackTrace(writer); - jsonWriter.writeString(writer.getBuffer(), 0, writer.getPosition()); + truncate(writer); + jsonWriter.writeString(writer.buffer(), 0, writer.position()); } finally { writerRecycler.release(writer); } } + private void truncate(final TruncatingBufferedPrintWriter writer) { + + // Short-circuit if truncation is not enabled. + if (!truncationEnabled) { + return; + } + + // Check for string matches. + // noinspection ForLoopReplaceableByForEach (avoid iterator allocation) + for (int i = 0; i < truncationPointMatcherStrings.size(); i++) { + final String matcher = truncationPointMatcherStrings.get(i); + final int matchIndex = writer.indexOf(matcher); + if (matchIndex > 0) { + final int truncationPointIndex = matchIndex + matcher.length(); + truncate(writer, truncationPointIndex); + return; + } + } + + // Check for regex matches. + // noinspection ForLoopReplaceableByForEach (avoid iterator allocation) + for (int i = 0; i < groupedTruncationPointMatcherRegexes.size(); i++) { + final Pattern pattern = groupedTruncationPointMatcherRegexes.get(i); + final Matcher matcher = pattern.matcher(writer); + final boolean matched = matcher.matches(); + if (matched) { + final int lastGroup = matcher.groupCount(); + final int truncationPointIndex = matcher.start(lastGroup); + truncate(writer, truncationPointIndex); + return; + } + } + + } + + private void truncate( + final TruncatingBufferedPrintWriter writer, + final int index) { + writer.position(index); + writer.print(truncatedStringSuffix); + } + } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/MapAccessor.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/MapAccessor.java index 3893f50..3deb2da 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/MapAccessor.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/MapAccessor.java @@ -17,6 +17,7 @@ package org.apache.logging.log4j.layout.template.json.util; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -71,10 +72,58 @@ public class MapAccessor { } public boolean exists(final String[] path) { - final Object value = getObject(path, Object.class); + final Object value = getObject(path); return value != null; } + public <E> List<E> getList(final String key, final Class<E> clazz) { + final String[] path = {key}; + return getList(path, clazz); + } + + public <E> List<E> getList(final String[] path, final Class<E> clazz) { + + // Access the object. + final Object value = getObject(path); + if (value == null) { + return null; + } + + // Check the type. + if (!(value instanceof List)) { + final String message = String.format( + "was expecting a List<%s> at path %s: %s (of type %s)", + clazz, + Arrays.asList(path), + value, + value.getClass().getCanonicalName()); + throw new IllegalArgumentException(message); + } + + // Check the element types. + @SuppressWarnings("unchecked") + final List<Object> items = (List<Object>) value; + for (int itemIndex = 0; itemIndex < items.size(); itemIndex++) { + final Object item = items.get(itemIndex); + if (!clazz.isInstance(item)) { + final String message = String.format( + "was expecting a List<%s> item at path %s and index %d: %s (of type %s)", + clazz, + Arrays.asList(path), + itemIndex, + item, + item != null ? item.getClass().getCanonicalName() : null); + throw new IllegalArgumentException(message); + } + } + + // Return the typed list. + @SuppressWarnings("unchecked") + final List<E> typedItems = (List<E>) items; + return typedItems; + + } + public Object getObject(final String key) { final String[] path = {key}; return getObject(path, Object.class); diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedPrintWriter.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedPrintWriter.java index 8d7cb1e..7e9aa3c 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedPrintWriter.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedPrintWriter.java @@ -17,8 +17,11 @@ package org.apache.logging.log4j.layout.template.json.util; import java.io.PrintWriter; +import java.util.Objects; -public final class TruncatingBufferedPrintWriter extends PrintWriter { +public final class TruncatingBufferedPrintWriter + extends PrintWriter + implements CharSequence { private final TruncatingBufferedWriter writer; @@ -36,20 +39,44 @@ public final class TruncatingBufferedPrintWriter extends PrintWriter { return new TruncatingBufferedPrintWriter(writer); } - public char[] getBuffer() { - return writer.getBuffer(); + public char[] buffer() { + return writer.buffer(); } - public int getPosition() { - return writer.getPosition(); + public int position() { + return writer.position(); } - public int getCapacity() { - return writer.getCapacity(); + public void position(final int index) { + writer.position(index); } - public boolean isTruncated() { - return writer.isTruncated(); + public int capacity() { + return writer.capacity(); + } + + public boolean truncated() { + return writer.truncated(); + } + + public int indexOf(final CharSequence seq) { + Objects.requireNonNull(seq, "seq"); + return writer.indexOf(seq); + } + + @Override + public int length() { + return writer.length(); + } + + @Override + public char charAt(final int index) { + return writer.charAt(index); + } + + @Override + public CharSequence subSequence(final int startIndex, final int endIndex) { + return writer.subSequence(startIndex, endIndex); } @Override @@ -57,4 +84,9 @@ public final class TruncatingBufferedPrintWriter extends PrintWriter { writer.close(); } + @Override + public String toString() { + return writer.toString(); + } + } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriter.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriter.java index ea50f77..1b88f12 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriter.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriter.java @@ -19,7 +19,7 @@ package org.apache.logging.log4j.layout.template.json.util; import java.io.Writer; import java.util.Objects; -public final class TruncatingBufferedWriter extends Writer { +final class TruncatingBufferedWriter extends Writer implements CharSequence { private final char[] buffer; @@ -33,19 +33,26 @@ public final class TruncatingBufferedWriter extends Writer { this.truncated = false; } - char[] getBuffer() { + char[] buffer() { return buffer; } - int getPosition() { + int position() { return position; } - int getCapacity() { + void position(final int index) { + if (index < 0 || index >= buffer.length) { + throw new IllegalArgumentException("invalid index: " + index); + } + position = index; + } + + int capacity() { return buffer.length; } - boolean isTruncated() { + boolean truncated() { return truncated; } @@ -196,6 +203,53 @@ public final class TruncatingBufferedWriter extends Writer { } + int indexOf(final CharSequence seq) { + + // Short-circuit if there is nothing to match. + final int seqLength = seq.length(); + if (seqLength == 0) { + return 0; + } + + // Short-circuit if the given input is longer than the buffer. + if (seqLength > position) { + return -1; + } + + // Perform the search. + for (int bufferIndex = 0; bufferIndex < position; bufferIndex++) { + boolean found = true; + for (int seqIndex = 0; seqIndex < seqLength; seqIndex++) { + final char s = seq.charAt(seqIndex); + final char b = buffer[bufferIndex + seqIndex]; + if (s != b) { + found = false; + break; + } + } + if (found) { + return bufferIndex; + } + } + return -1; + + } + + @Override + public int length() { + return position + 1; + } + + @Override + public char charAt(final int index) { + return buffer[index]; + } + + @Override + public String subSequence(final int startIndex, final int endIndex) { + return new String(buffer, startIndex, endIndex - startIndex); + } + @Override public void flush() {} @@ -205,4 +259,9 @@ public final class TruncatingBufferedWriter extends Writer { truncated = false; } + @Override + public String toString() { + return new String(buffer, 0, position); + } + } diff --git a/log4j-layout-template-json/src/main/resources/EcsLayout.json b/log4j-layout-template-json/src/main/resources/EcsLayout.json index dee7a84..708b27b 100644 --- a/log4j-layout-template-json/src/main/resources/EcsLayout.json +++ b/log4j-layout-template-json/src/main/resources/EcsLayout.json @@ -41,6 +41,8 @@ "error.stack_trace": { "$resolver": "exception", "field": "stackTrace", - "stringified": true + "stackTrace": { + "stringified": true + } } } diff --git a/log4j-layout-template-json/src/main/resources/GelfLayout.json b/log4j-layout-template-json/src/main/resources/GelfLayout.json index dd43cc8..4281bba 100644 --- a/log4j-layout-template-json/src/main/resources/GelfLayout.json +++ b/log4j-layout-template-json/src/main/resources/GelfLayout.json @@ -8,7 +8,9 @@ "full_message": { "$resolver": "exception", "field": "stackTrace", - "stringified": true + "stackTrace": { + "stringified": true + } }, "timestamp": { "$resolver": "timestamp", diff --git a/log4j-layout-template-json/src/main/resources/LogstashJsonEventLayoutV1.json b/log4j-layout-template-json/src/main/resources/LogstashJsonEventLayoutV1.json index 3225930..809f705 100644 --- a/log4j-layout-template-json/src/main/resources/LogstashJsonEventLayoutV1.json +++ b/log4j-layout-template-json/src/main/resources/LogstashJsonEventLayoutV1.json @@ -15,7 +15,9 @@ "stacktrace": { "$resolver": "exception", "field": "stackTrace", - "stringified": true + "stackTrace": { + "stringified": true + } } }, "line_number": { diff --git a/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayoutTest.java b/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayoutTest.java index b184ca8..92b07ec 100644 --- a/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayoutTest.java +++ b/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/JsonTemplateLayoutTest.java @@ -473,7 +473,8 @@ class JsonTemplateLayoutTest { "ex_stacktrace", asMap( "$resolver", "exception", "field", "stackTrace", - "stringified", true), + "stackTrace", asMap( + "stringified", true)), "root_ex_class", asMap( "$resolver", "exceptionRootCause", "field", "className"), @@ -483,7 +484,8 @@ class JsonTemplateLayoutTest { "root_ex_stacktrace", asMap( "$resolver", "exceptionRootCause", "field", "stackTrace", - "stringified", true))); + "stackTrace", asMap( + "stringified", true)))); // Create the layout. final JsonTemplateLayout layout = JsonTemplateLayout @@ -547,7 +549,8 @@ class JsonTemplateLayoutTest { "root_ex_stacktrace", asMap( "$resolver", "exceptionRootCause", "field", "stackTrace", - "stringified", true))); + "stackTrace", asMap( + "stringified", true)))); // Create the layout. final JsonTemplateLayout layout = JsonTemplateLayout @@ -1604,14 +1607,16 @@ class JsonTemplateLayoutTest { "exStackTraceString", asMap( "$resolver", "exception", "field", "stackTrace", - "stringified", true), + "stackTrace", asMap( + "stringified", true)), "exRootCauseStackTrace", asMap( "$resolver", "exceptionRootCause", "field", "stackTrace"), "exRootCauseStackTraceString", asMap( "$resolver", "exceptionRootCause", "field", "stackTrace", - "stringified", true), + "stackTrace", asMap( + "stringified", true)), "requiredFieldTriggeringError", true)); // Create the layout. @@ -1634,7 +1639,7 @@ class JsonTemplateLayoutTest { } @Test - void test_StackTraceTextResolver_with_maxStringLength() { + void test_stringified_exception_resolver_with_maxStringLength() { // Create the event template. final String eventTemplate = writeJson(asMap( @@ -1672,6 +1677,119 @@ class JsonTemplateLayoutTest { } @Test + void test_stack_trace_truncation() { + + // Create the exception to be logged. + final Exception childError = + new Exception("unique child exception message"); + final Exception parentError = + new Exception("unique parent exception message", childError); + + // Create the event template. + final String truncatedStringSuffix = "~"; + final String eventTemplate = writeJson(asMap( + // Raw exception. + "ex", asMap( + "$resolver", "exception", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true)), + // Exception matcher using strings. + "stringMatchedEx", asMap( + "$resolver", "exception", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true, + "truncatedStringSuffix", truncatedStringSuffix, + "truncationPointMatcherStrings", Arrays.asList( + "this string shouldn't match with anything", + parentError.getMessage()))), + // Exception matcher using regexes. + "regexMatchedEx", asMap( + "$resolver", "exception", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true, + "truncatedStringSuffix", truncatedStringSuffix, + "truncationPointMatcherRegexes", Arrays.asList( + "this string shouldn't match with anything", + parentError + .getMessage() + .replace("unique", "[xu]n.que")))), + // Raw exception root cause. + "rootEx", asMap( + "$resolver", "exceptionRootCause", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true)), + // Exception root cause matcher using strings. + "stringMatchedRootEx", asMap( + "$resolver", "exceptionRootCause", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true, + "truncatedStringSuffix", truncatedStringSuffix, + "truncationPointMatcherStrings", Arrays.asList( + "this string shouldn't match with anything", + childError.getMessage()))), + // Exception root cause matcher using regexes. + "regexMatchedRootEx", asMap( + "$resolver", "exceptionRootCause", + "field", "stackTrace", + "stackTrace", asMap( + "stringified", true, + "truncatedStringSuffix", truncatedStringSuffix, + "truncationPointMatcherRegexes", Arrays.asList( + "this string shouldn't match with anything", + childError + .getMessage() + .replace("unique", "[xu]n.que")))))); + + // Create the layout. + final JsonTemplateLayout layout = JsonTemplateLayout + .newBuilder() + .setConfiguration(CONFIGURATION) + .setEventTemplate(eventTemplate) + .setStackTraceEnabled(true) + .build(); + + // Create the log event. + final LogEvent logEvent = Log4jLogEvent + .newBuilder() + .setLoggerName(LOGGER_NAME) + .setThrown(parentError) + .build(); + + // Check the serialized event. + final String expectedMatchedExEnd = + parentError.getMessage() + truncatedStringSuffix; + final String expectedMatchedRootExEnd = + childError.getMessage() + truncatedStringSuffix; + usingSerializedLogEventAccessor(layout, logEvent, accessor -> { + + // Check the serialized exception. + assertThat(accessor.getString("ex")) + .doesNotEndWith(expectedMatchedExEnd) + .doesNotEndWith(expectedMatchedRootExEnd); + assertThat(accessor.getString("stringMatchedEx")) + .endsWith(expectedMatchedExEnd); + assertThat(accessor.getString("regexMatchedEx")) + .endsWith(expectedMatchedExEnd); + + // Check the serialized exception root cause. + assertThat(accessor.getString("rootEx")) + .doesNotEndWith(expectedMatchedExEnd) + .doesNotEndWith(expectedMatchedRootExEnd); + assertThat(accessor.getString("stringMatchedRootEx")) + .endsWith(expectedMatchedRootExEnd); + assertThat(accessor.getString("regexMatchedRootEx")) + .endsWith(expectedMatchedRootExEnd); + + }); + + } + + @Test void test_null_eventDelimiter() { // Create the event template. diff --git a/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriterTest.java b/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriterTest.java index a8b210c..b52d453 100644 --- a/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriterTest.java +++ b/log4j-layout-template-json/src/test/java/org/apache/logging/log4j/layout/template/json/util/TruncatingBufferedWriterTest.java @@ -75,10 +75,10 @@ class TruncatingBufferedWriterTest { expectedBuffer[expectedPosition++] = 'u'; expectedBuffer[expectedPosition++] = 'l'; expectedBuffer[expectedPosition++] = 'l'; - Assertions.assertThat(writer.getBuffer()).isEqualTo(expectedBuffer); - Assertions.assertThat(writer.getPosition()).isEqualTo(expectedPosition); - Assertions.assertThat(writer.getCapacity()).isEqualTo(capacity); - Assertions.assertThat(writer.isTruncated()).isFalse(); + Assertions.assertThat(writer.buffer()).isEqualTo(expectedBuffer); + Assertions.assertThat(writer.position()).isEqualTo(expectedPosition); + Assertions.assertThat(writer.capacity()).isEqualTo(capacity); + Assertions.assertThat(writer.truncated()).isFalse(); verifyClose(writer); } @@ -228,17 +228,17 @@ class TruncatingBufferedWriterTest { private void verifyTruncation( final TruncatingBufferedWriter writer, final char c) { - Assertions.assertThat(writer.getBuffer()).isEqualTo(new char[]{c}); - Assertions.assertThat(writer.getPosition()).isEqualTo(1); - Assertions.assertThat(writer.getCapacity()).isEqualTo(1); - Assertions.assertThat(writer.isTruncated()).isTrue(); + Assertions.assertThat(writer.buffer()).isEqualTo(new char[]{c}); + Assertions.assertThat(writer.position()).isEqualTo(1); + Assertions.assertThat(writer.capacity()).isEqualTo(1); + Assertions.assertThat(writer.truncated()).isTrue(); verifyClose(writer); } private void verifyClose(final TruncatingBufferedWriter writer) { writer.close(); - Assertions.assertThat(writer.getPosition()).isEqualTo(0); - Assertions.assertThat(writer.isTruncated()).isFalse(); + Assertions.assertThat(writer.position()).isEqualTo(0); + Assertions.assertThat(writer.truncated()).isFalse(); } } diff --git a/log4j-layout-template-json/src/test/resources/testJsonTemplateLayout.json b/log4j-layout-template-json/src/test/resources/testJsonTemplateLayout.json index daf455e..e8e1063 100644 --- a/log4j-layout-template-json/src/test/resources/testJsonTemplateLayout.json +++ b/log4j-layout-template-json/src/test/resources/testJsonTemplateLayout.json @@ -10,7 +10,9 @@ "stacktrace": { "$resolver": "exception", "field": "stackTrace", - "stringified": true + "stackTrace": { + "stringified": true + } }, "line_number": { "$resolver": "source", diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 5264b56..a248808 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -30,6 +30,9 @@ - "remove" - Removed --> <release version="2.14.1" date="2021-MM-DD" description="GA Release 2.14.1"> + <action issue="LOG4J2-2993" dev="vy" type="add"> + Support stack trace truncation in JsonTemplateLayout. + </action> <action issue="LOG4J2-2998" dev="vy" type="fix"> Fix truncation of excessive strings ending with a high surrogate in JsonWriter. </action> diff --git a/src/site/asciidoc/manual/json-template-layout.adoc.vm b/src/site/asciidoc/manual/json-template-layout.adoc.vm index 6003ffe..47d4511 100644 --- a/src/site/asciidoc/manual/json-template-layout.adoc.vm +++ b/src/site/asciidoc/manual/json-template-layout.adoc.vm @@ -434,20 +434,43 @@ a| a| [source] ---- -config = field , [ stringified ] -field = "field" -> ( - "className" \| - "message" \| - "stackTrace" ) -stringified = "stringified" -> boolean +config = field , [ stringified ] , [ stackTrace ] +field = "field" -> ( "className" \| "message" \| "stackTrace" ) +stackTrace = "stackTrace" -> ( + [ stringified ] + , [ truncatedStringSuffix ] + , [ truncationPointMatcherStrings ] + , [ truncationPointMatcherRegexes ] + ) +stringified = "stringified" -> boolean +truncatedStringSuffix = "truncatedStringSuffix" -> string +truncationPointMatcherStrings = "truncationPointMatcherStrings" -> string[] +truncationPointMatcherRegexes = "truncationPointMatcherRegexes" -> string[] ---- a| Resolves fields of the `Throwable` returned by `logEvent.getThrown()`. +`stringified` is set to `false` by default. `stringified` at the root level is +*deprecated*, instead prefer the one in the `stackTrace` object, which has +precedence if both are provided. + +`truncationPointMatcherStrings` and `truncationPointMatcherRegexes` enable the +truncation of the stack trace after the given matching point. If both parameters +are provided, `truncationPointMatcherStrings` will be checked first. Note that +these configurations are only taken into account when `stringified` is set to +`true`. + +`truncatedStringSuffix` will be set to the one configured in the layout, unless +explicitly provided. + Note that this resolver is toggled by `log4j.layout.jsonTemplate.stackTraceEnabled` property. -| Since `Throwable#getStackTrace()` clones the original `StackTraceElement[]`, - access to (and hence rendering of) stack traces are not garbage-free. +a| +Since `Throwable#getStackTrace()` clones the original `StackTraceElement[]`, +access to (and hence rendering of) stack traces are not garbage-free. + +Each `truncationPointMatcherRegexes` item triggers a `Pattern\#matcher()` call, +which is not garbage-free. a| Resolve `logEvent.getThrown().getClass().getCanonicalName()`: @@ -476,20 +499,35 @@ Resolve the stack trace into a string field: { "$resolver": "exception", "field": "stackTrace", - "stringified": true + "stackTrace": { + "stringified": true + } } ---- -| exceptionRootCause -| identical to `exception` resolver -a| -Resolves the fields of the innermost `Throwable` returned by -`logEvent.getThrown()`. +Resolve the stack trace into a string field such that the content will be +truncated by the given points: -Note that this resolver is toggled by -`log4j.layout.jsonTemplate.stackTraceEnabled` property. +[source,json] +---- +{ + "$resolver": "exception", + "field": "stackTrace", + "stackTrace": { + "stringified": true, + "truncatedStringSuffix": ">", + "truncationPointStrings": ["at javax.servlet.http.HttpServlet.service"] + } +} +---- + +| exceptionRootCause | identical to `exception` resolver +| identical to `exception` resolver with the exception that the innermost + `Throwable` in the causal-chain of `logEvent.getThrown()` is resolved | identical to `exception` resolver +| identical to `exception` resolver with the exception that `${dollar}resolver` + field needs to be set to `exceptionRootCause` | level a|
