This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 449fe78dd5 BP-69: Print slog context attrs in log4j2; add OTel JSON
example (#4770)
449fe78dd5 is described below
commit 449fe78dd51f2eb3e73e3c5d577851ff18144025
Author: Matteo Merli <[email protected]>
AuthorDate: Tue May 5 00:51:57 2026 -0700
BP-69: Print slog context attrs in log4j2; add OTel JSON example (#4770)
* Print structured attributes in log4j2 patterns; add OTel JSON example
Slog attributes are forwarded to log4j2 as ContextData (the same channel
SLF4J's MDC uses). Add %X to every PatternLayout in the user-facing
log4j2 configs so the structured attributes attached via .attr(...)
appear in log output by default. Drop the now-redundant *MDC variant
appenders and the directentrylogger override that targeted them.
Also add conf/log4j2.otel-json.xml plus conf/otel-log-template.json as
an example layout that emits each event as a JSON object aligned with
the OpenTelemetry log data model (severity_text, severity_number, body,
attributes, resource, instrumentation_scope, trace_id/span_id from MDC).
The example uses log4j-layout-template-json, which must be added to the
classpath alongside log4j-core to enable JsonTemplateLayout.
* log4j2: render context map after the message
* Align OTel JSON template with the OpenTelemetry log data model
Replace the previous template with the structure used by Pulsar
(apache/pulsar/blob/master/conf/OtelLogLayout.json):
* Top-level fields use the spec's PascalCase naming (Timestamp,
SeverityText, Body, TraceId, SpanId, TraceFlags, Attributes).
* thread.name, thread.id, code.namespace, exception.{type,message,
stacktrace}, and the flattened MDC are placed under "Attributes",
using OTel semantic-convention keys.
* Drop SeverityNumber, ObservedTimestamp, InstrumentationScope, and
Resource — the first two are derivable / redundant for app-emitted
logs, the logger name is "code.namespace" not InstrumentationScope,
and Resource is normally attached by the collector pipeline.
* The MDC resolver with flatten=true now sits inside Attributes, so
slog .attr(...) keys merge as siblings of thread.name etc. (the
previous placement at the top level didn't merge into "attributes"
as intended).
Update the explanatory comment block and drop the now-unused
otel.service.name / otel.host.name properties.
---
bookkeeper-benchmark/conf/log4j2.xml | 6 +-
bookkeeper-server/src/main/resources/log4j2.xml | 2 +-
conf/log4j2.cli.xml | 2 +-
conf/log4j2.otel-json.xml | 86 +++++++++++++++++++++++++
conf/log4j2.shell.xml | 2 +-
conf/log4j2.xml | 29 +++------
conf/otel-log-template.json | 63 ++++++++++++++++++
stream/conf/log4j2.cli.xml | 6 +-
stream/conf/log4j2.xml | 6 +-
9 files changed, 171 insertions(+), 31 deletions(-)
diff --git a/bookkeeper-benchmark/conf/log4j2.xml
b/bookkeeper-benchmark/conf/log4j2.xml
index 21257d6dc3..e89054596e 100644
--- a/bookkeeper-benchmark/conf/log4j2.xml
+++ b/bookkeeper-benchmark/conf/log4j2.xml
@@ -22,13 +22,13 @@
<Configuration status="warn" monitorInterval="10">
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
</Console>
<File name="TRACEFILE" fileName="bookkeeper_trace.log">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m %X%n"/>
</File>
<RollingFile name="ROLLINGFILE" fileName="bookkeeper-benchmark.log"
filePattern="bookkeeper-benchmark.log%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
diff --git a/bookkeeper-server/src/main/resources/log4j2.xml
b/bookkeeper-server/src/main/resources/log4j2.xml
index 3e3e4bc8c8..b67f9faa5d 100644
--- a/bookkeeper-server/src/main/resources/log4j2.xml
+++ b/bookkeeper-server/src/main/resources/log4j2.xml
@@ -22,7 +22,7 @@
<Configuration status="warn" monitorInterval="10">
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/conf/log4j2.cli.xml b/conf/log4j2.cli.xml
index 3a1d2fa4ff..6f2b2bf1b7 100644
--- a/conf/log4j2.cli.xml
+++ b/conf/log4j2.cli.xml
@@ -34,7 +34,7 @@
<PatternLayout pattern="%m%n"/>
</Console>
<RollingFile name="ROLLINGFILE"
fileName="${sys:bookkeeper.cli.log.dir}/${sys:bookkeeper.cli.log.file}"
filePattern="${sys:bookkeeper.cli.log.dir}/${sys:bookkeeper.cli.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
diff --git a/conf/log4j2.otel-json.xml b/conf/log4j2.otel-json.xml
new file mode 100644
index 0000000000..a6df4c4735
--- /dev/null
+++ b/conf/log4j2.otel-json.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!--
+ Example log4j2 configuration that emits each log event as a single JSON
+ object aligned with the OpenTelemetry log data model
+ (https://opentelemetry.io/docs/specs/otel/logs/data-model/).
+
+ Required runtime dependency (NOT included by default in BookKeeper):
+ org.apache.logging.log4j:log4j-layout-template-json:${log4j.version}
+
+ Add it to the classpath alongside log4j-core to enable JsonTemplateLayout.
+
+ The event template lives in conf/otel-log-template.json. Customize it
+ there (e.g. add resource attributes, drop fields, rename keys) without
+ touching this file.
+
+ Top-level fields follow the OTel data-model field naming (PascalCase):
+ * "Timestamp" = event time, ISO-8601 UTC.
+ * "SeverityText" = log4j level name ("INFO", "WARN", ...).
+ * "Body" = the log message string.
+ * "TraceId" / "SpanId" / "TraceFlags"
+ = MDC keys conventionally set by the
+ OpenTelemetry log appender bridge; omitted
+ when absent.
+ * "Attributes" = OTel-semantic-conventions attributes for the
+ event:
+ - thread.name / thread.id
+ - code.namespace (= the logger name)
+ - exception.type / .message / .stacktrace
+ Plus the full Log4j2 ContextData map (every
+ slog ".attr(...)" entry and any SLF4J MDC
+ key) merged in via the "mdc" resolver with
+ flatten=true.
+-->
+<Configuration status="warn" monitorInterval="10">
+ <Properties>
+ <Property name="bookkeeper.log.dir">.</Property>
+ <Property
name="bookkeeper.log.file">bookkeeper-server.json.log</Property>
+ <Property name="bookkeeper.log.root.level">INFO</Property>
+ <Property name="bookkeeper.log.root.appender">CONSOLE</Property>
+ <Property
name="otel.template">file:${sys:bookkeeper.conf.dir:-conf}/otel-log-template.json</Property>
+ </Properties>
+ <Appenders>
+ <Console name="CONSOLE" target="SYSTEM_OUT">
+ <JsonTemplateLayout eventTemplateUri="${otel.template}"/>
+ </Console>
+ <RollingFile name="ROLLINGFILE"
+
fileName="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}"
+
filePattern="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}%d{.yyyy-MM-dd}">
+ <!--
+ Each line is an independent JSON object (ndjson), directly
+ consumable by log shippers (FluentBit, Vector, Fluentd,
+ Promtail, otel-collector, ...).
+ -->
+ <JsonTemplateLayout eventTemplateUri="${otel.template}"/>
+ <Policies>
+ <TimeBasedTriggeringPolicy modulate="true"/>
+ </Policies>
+ <DefaultRolloverStrategy max="100"/>
+ </RollingFile>
+ </Appenders>
+ <Loggers>
+ <Root level="${sys:bookkeeper.log.root.level}">
+ <AppenderRef ref="${sys:bookkeeper.log.root.appender}"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/conf/log4j2.shell.xml b/conf/log4j2.shell.xml
index c6e6b0fb9c..563149697a 100644
--- a/conf/log4j2.shell.xml
+++ b/conf/log4j2.shell.xml
@@ -31,7 +31,7 @@
<PatternLayout pattern="%d{ABSOLUTE} %-5p %m%n"/>
</Console>
<RollingFile name="ROLLINGFILE"
fileName="${sys:bookkeeper.shell.log.dir}/${sys:bookkeeper.shell.log.file}"
filePattern="${sys:bookkeeper.shell.log.dir}/${sys:bookkeeper.shell.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
diff --git a/conf/log4j2.xml b/conf/log4j2.xml
index 08d5d5613c..8aed8f0dab 100644
--- a/conf/log4j2.xml
+++ b/conf/log4j2.xml
@@ -27,27 +27,21 @@
<Property name="bookkeeper.log.root.appender">CONSOLE</Property>
</Properties>
<Appenders>
+ <!--
+ The "%X" pattern element renders the structured attributes attached
+ to each log event (slog ".attr(...)" calls and any java.util.MDC /
+ Log4j2 ThreadContext entries). It is rendered as "{key=value,
...}".
+ Drop or replace it with named keys (e.g. %X{ledgerId}) for a more
+ specific format.
+ -->
<Console name="CONSOLE" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
- </Console>
- <Console name="CONSOLEMDC" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] %X -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
</Console>
<File name="TRACEFILE"
fileName="${sys:bookkeeper.log.dir}/bookkeeper-trace.log">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m%n"/>
- </File>
- <File name="TRACEFILEMDC"
fileName="${sys:bookkeeper.log.dir}/bookkeeper-trace.log">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] %X
- %m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m %X%n"/>
</File>
<RollingFile name="ROLLINGFILE"
fileName="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}"
filePattern="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"/>
- <Policies>
- <TimeBasedTriggeringPolicy modulate="true"/>
- </Policies>
- <DefaultRolloverStrategy max="100"/>
- </RollingFile>
- <RollingFile name="ROLLINGFILEMDC"
fileName="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}"
filePattern="${sys:bookkeeper.log.dir}/${sys:bookkeeper.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] %X -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
@@ -58,8 +52,5 @@
<Root level="${sys:bookkeeper.log.root.level}">
<AppenderRef ref="${sys:bookkeeper.log.root.appender}"/>
</Root>
- <Logger name="org.apache.bookkeeper.bookie.storage.directentrylogger"
level="INFO">
- <AppenderRef ref="${sys:bookkeeper.log.root.appender}MDC"/>
- </Logger>
</Loggers>
</Configuration>
diff --git a/conf/otel-log-template.json b/conf/otel-log-template.json
new file mode 100644
index 0000000000..d3e8e95f85
--- /dev/null
+++ b/conf/otel-log-template.json
@@ -0,0 +1,63 @@
+{
+ "Timestamp": {
+ "$resolver": "timestamp",
+ "pattern": {
+ "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
+ "timeZone": "UTC"
+ }
+ },
+ "SeverityText": {
+ "$resolver": "level",
+ "field": "name"
+ },
+ "Body": {
+ "$resolver": "message",
+ "stringified": true
+ },
+ "TraceId": {
+ "$resolver": "mdc",
+ "key": "trace_id"
+ },
+ "SpanId": {
+ "$resolver": "mdc",
+ "key": "span_id"
+ },
+ "TraceFlags": {
+ "$resolver": "mdc",
+ "key": "trace_flags"
+ },
+ "Attributes": {
+ "thread.name": {
+ "$resolver": "thread",
+ "field": "name"
+ },
+ "thread.id": {
+ "$resolver": "thread",
+ "field": "id"
+ },
+ "code.namespace": {
+ "$resolver": "logger",
+ "field": "name"
+ },
+ "exception.type": {
+ "$resolver": "exception",
+ "field": "className"
+ },
+ "exception.message": {
+ "$resolver": "exception",
+ "field": "message"
+ },
+ "exception.stacktrace": {
+ "$resolver": "exception",
+ "field": "stackTrace",
+ "stackTrace": {
+ "stringified": true
+ }
+ },
+ "mdc": {
+ "$resolver": "mdc",
+ "flatten": true,
+ "stringified": true
+ }
+ }
+}
diff --git a/stream/conf/log4j2.cli.xml b/stream/conf/log4j2.cli.xml
index 8c62501392..e29f733128 100644
--- a/stream/conf/log4j2.cli.xml
+++ b/stream/conf/log4j2.cli.xml
@@ -28,13 +28,13 @@
</Properties>
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
</Console>
<File name="TRACEFILE"
fileName="${sys:streamstorage.log.dir}/streamstorage-trace.log">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m %X%n"/>
</File>
<RollingFile name="ROLLINGFILE"
fileName="${sys:streamstorage.log.dir}/${sys:streamstorage.log.file}"
filePattern="${sys:streamstorage.log.dir}/${sys:streamstorage.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
diff --git a/stream/conf/log4j2.xml b/stream/conf/log4j2.xml
index 006e4bf697..fd16c99e49 100644
--- a/stream/conf/log4j2.xml
+++ b/stream/conf/log4j2.xml
@@ -28,13 +28,13 @@
</Properties>
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m
%X%n"/>
</Console>
<File name="TRACEFILE"
fileName="${sys:streamstorage.log.dir}/streamstorage-trace.log">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L][%ndc] -
%m %X%n"/>
</File>
<RollingFile name="ROLLINGFILE"
fileName="${sys:streamstorage.log.dir}/${sys:streamstorage.log.file}"
filePattern="${sys:streamstorage.log.dir}/${sys:streamstorage.log.file}%d{.yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"/>
+ <PatternLayout pattern="%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m
%X%n"/>
<Policies>
<TimeBasedTriggeringPolicy modulate="true"/>
</Policies>
