This is an automated email from the ASF dual-hosted git repository. ramanathan1504 pushed a commit to branch fix/issue-4166 in repository https://gitbox.apache.org/repos/asf/logging-log4j2.git
commit d59c38bad538daddafd8a973d6eda8f8b8b4bf3b Author: Ramanathan <[email protected]> AuthorDate: Tue Jul 21 13:12:58 2026 +0530 add keyExcludes to MDC resolvers to prevent overwriting of structural log fields --- .../resolver/ReadOnlyStringMapResolverTest.java | 32 +++++++++ .../json/resolver/ReadOnlyStringMapResolver.java | 16 ++++- .../src/main/resources/EcsLayout.json | 12 +++- .../log4j/perf/jmh/MdcKeyFilterBenchmark.java | 81 ++++++++++++++++++++++ src/changelog/.2.x.x/4166_add_mdc_keyExcludes.xml | 13 ++++ .../ROOT/pages/manual/json-template-layout.adoc | 12 ++++ 6 files changed, 164 insertions(+), 2 deletions(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolverTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolverTest.java index 422514e9f4..cf1a98d489 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolverTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolverTest.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.List; import java.util.regex.PatternSyntaxException; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.DefaultConfiguration; import org.apache.logging.log4j.core.impl.Log4jLogEvent; import org.apache.logging.log4j.layout.template.json.JsonTemplateLayout; import org.apache.logging.log4j.message.Message; @@ -386,4 +387,35 @@ class ReadOnlyStringMapResolverTest { assertThat(accessor.getString("stringifiedValue")).isEqualTo(String.valueOf(value)); }); } + + @Test + void test_keyExcludes() { + + final String eventTemplate = "" + "{\n" + + " \"$resolver\": \"mdc\",\n" + + " \"keyExcludes\": [\"@timestamp\", \"message\", \"log.logger\"]\n" + + "}"; + + final StringMap contextData = new SortedArrayStringMap(); + contextData.putValue("allowedKey1", "value1"); + contextData.putValue("message", "this should be hidden"); + contextData.putValue("@timestamp", "this should also be hidden"); + contextData.putValue("allowedKey2", "value2"); + + final LogEvent logEvent = + Log4jLogEvent.newBuilder().setContextData(contextData).build(); + + final JsonTemplateLayout layout = JsonTemplateLayout.newBuilder() + .setConfiguration(new DefaultConfiguration()) + .setEventTemplate(eventTemplate) + .build(); + + final String serializedJson = layout.toSerializable(logEvent); + + assertThat(serializedJson).contains("\"allowedKey1\":\"value1\""); + assertThat(serializedJson).contains("\"allowedKey2\":\"value2\""); + assertThat(serializedJson).doesNotContain("\"message\""); + assertThat(serializedJson).doesNotContain("\"@timestamp\""); + assertThat(serializedJson).doesNotContain("this should be hidden"); + } } diff --git a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolver.java b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolver.java index 144525a06e..13fd52dbc5 100644 --- a/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolver.java +++ b/log4j-layout-template-json/src/main/java/org/apache/logging/log4j/layout/template/json/resolver/ReadOnlyStringMapResolver.java @@ -16,7 +16,10 @@ */ package org.apache.logging.log4j.layout.template.json.resolver; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -204,12 +207,16 @@ class ReadOnlyStringMapResolver implements EventResolver { if (pattern == null && replacement != null) { throw new IllegalArgumentException("replacement cannot be provided without a pattern: " + config); } + final List<String> keyExcludesList = config.getList("keyExcludes", String.class); + final Set<String> keyExcludes = + keyExcludesList == null || keyExcludesList.isEmpty() ? null : new HashSet<>(keyExcludesList); final boolean stringified = config.getBoolean("stringified", false); if (key != null) { return createKeyResolver(key, stringified, mapAccessor); } else { final RecyclerFactory recyclerFactory = context.getRecyclerFactory(); - return createResolver(recyclerFactory, flatten, prefix, pattern, replacement, stringified, mapAccessor); + return createResolver( + recyclerFactory, flatten, prefix, pattern, replacement, keyExcludes, stringified, mapAccessor); } } @@ -243,6 +250,7 @@ class ReadOnlyStringMapResolver implements EventResolver { final String prefix, final String pattern, final String replacement, + final Set<String> keyExcludes, final boolean stringified, final Function<LogEvent, ReadOnlyStringMap> mapAccessor) { @@ -258,6 +266,7 @@ class ReadOnlyStringMapResolver implements EventResolver { } loopContext.pattern = compiledPattern; loopContext.replacement = replacement; + loopContext.keyExcludes = keyExcludes; loopContext.stringified = stringified; return loopContext; }); @@ -331,6 +340,8 @@ class ReadOnlyStringMapResolver implements EventResolver { private String replacement; + private Set<String> keyExcludes; + private boolean stringified; private JsonWriter jsonWriter; @@ -345,6 +356,9 @@ class ReadOnlyStringMapResolver implements EventResolver { @Override public void accept(final String key, final Object value, final LoopContext loopContext) { + if (loopContext.keyExcludes != null && loopContext.keyExcludes.contains(key)) { + return; + } final Matcher matcher = loopContext.pattern != null ? loopContext.pattern.matcher(key) : null; final boolean keyMatched = matcher == null || matcher.matches(); if (keyMatched) { diff --git a/log4j-layout-template-json/src/main/resources/EcsLayout.json b/log4j-layout-template-json/src/main/resources/EcsLayout.json index 8d215ab56a..1baae95fe1 100644 --- a/log4j-layout-template-json/src/main/resources/EcsLayout.json +++ b/log4j-layout-template-json/src/main/resources/EcsLayout.json @@ -26,7 +26,17 @@ "labels": { "$resolver": "mdc", "flatten": true, - "stringified": true + "stringified": true, + "keyExcludes": [ + "@timestamp", + "message", + "log.logger", + "log.level", + "event.dataset", + "process.thread.name", + "process.thread.id", + "ecs.version" + ] }, "tags": { "$resolver": "ndc" diff --git a/log4j-perf-test/src/main/java/org/apache/logging/log4j/perf/jmh/MdcKeyFilterBenchmark.java b/log4j-perf-test/src/main/java/org/apache/logging/log4j/perf/jmh/MdcKeyFilterBenchmark.java new file mode 100644 index 0000000000..4ca6f62c7f --- /dev/null +++ b/log4j-perf-test/src/main/java/org/apache/logging/log4j/perf/jmh/MdcKeyFilterBenchmark.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.logging.log4j.perf.jmh; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class MdcKeyFilterBenchmark { + + private static final String[] MDC_KEYS = { + "userId", "transactionId", "userRole", + "@timestamp", "message", "log.level" + }; + + private static final Pattern PATTERN = Pattern.compile( + "^(?!@timestamp|message|log\\.logger|log\\.level|event\\.dataset|process\\.thread\\.name|process\\.thread\\.id|ecs\\.version).*$"); + + private static final Set<String> EXCLUDED_KEYS = new HashSet<>(Arrays.asList( + "@timestamp", + "message", + "log.logger", + "log.level", + "event.dataset", + "process.thread.name", + "process.thread.id", + "ecs.version")); + + @Benchmark + public int testRegex() { + int allowedCount = 0; + for (String key : MDC_KEYS) { + if (PATTERN.matcher(key).matches()) { + allowedCount++; + } + } + return allowedCount; + } + + @Benchmark + public int testKeyExcludes() { + int allowedCount = 0; + for (String key : MDC_KEYS) { + if (!EXCLUDED_KEYS.contains(key)) { + allowedCount++; + } + } + return allowedCount; + } +} diff --git a/src/changelog/.2.x.x/4166_add_mdc_keyExcludes.xml b/src/changelog/.2.x.x/4166_add_mdc_keyExcludes.xml new file mode 100644 index 0000000000..c669bb9faf --- /dev/null +++ b/src/changelog/.2.x.x/4166_add_mdc_keyExcludes.xml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="UTF-8"?> +<entry xmlns="https://logging.apache.org/xml/ns"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation=" + https://logging.apache.org/xml/ns + https://logging.apache.org/xml/ns/log4j-changelog-0.xsd"; + type="added"> + <issue id="4166" link="https://github.com/apache/logging-log4j2/issues/4166"/> + <issue id="4186" link="https://github.com/apache/logging-log4j2/pull/4186"/> + <description format="asciidoc"> + Add `keyExcludes` to MDC resolvers to prevent user-controlled variables from overwriting structural log fields. + </description> +</entry> \ No newline at end of file diff --git a/src/site/antora/modules/ROOT/pages/manual/json-template-layout.adoc b/src/site/antora/modules/ROOT/pages/manual/json-template-layout.adoc index 8fc7b16de4..efc90759e6 100644 --- a/src/site/antora/modules/ROOT/pages/manual/json-template-layout.adoc +++ b/src/site/antora/modules/ROOT/pages/manual/json-template-layout.adoc @@ -1438,6 +1438,7 @@ stringified = "stringified" -> boolean multiAccess = [ pattern ] , [ replacement ] , [ flatten ] , [ stringified ] pattern = "pattern" -> string replacement = "replacement" -> string +keyExcludes = "keyExcludes" -> "[" , string , { "," , string } , "]" flatten = "flatten" -> ( boolean | flattenConfig ) flattenConfig = [ flattenPrefix ] flattenPrefix = "prefix" -> string @@ -1452,6 +1453,8 @@ Regex provided in the `pattern` is used to match against the keys. If provided, `replacement` will be used to replace the matched keys. These two are analogous to `Pattern.compile(pattern).matcher(key).matches()` and `Pattern.compile(pattern).matcher(key).replaceAll(replacement)` calls. +`keyExcludes` allows specifying keys that should be excluded from the resolution process. + [WARNING] ==== Regarding garbage footprint, `stringified` flag translates to `String.valueOf(value)`, hence mind values that are not `String`-typed. @@ -1526,6 +1529,15 @@ Resolve all fields whose keys match with the `user:(role|rank)` regex into an ob "replacement": "$1" } ---- +Resolve all fields into an object, but silently drop `message` and `log.level`: + +[source,json] +---- +{ + "$resolver": "…", + "keyExcludes": ["message", "log.level"] +} +---- Merge all fields whose keys are matching with the `user:(role|rank)` regex into the parent:
