From 0067315b312857783efa4c8d94b674cb7b113222 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 28 Apr 2026 13:00:18 -0700 Subject: [PATCH 1/3] Print structured attributes in log4j2 patterns; add OTel JSON example Slog attributes are forwarded to log4j2 as ContextData (the same channel SLF4J's MDC uses). Add %X to every PatternLayout in the user-facing log4j2 configs so the structured attributes attached via .attr(...) appear in log output by default. Drop the now-redundant *MDC variant appenders and the directentrylogger override that targeted them. Also add conf/log4j2.otel-json.xml plus conf/otel-log-template.json as an example layout that emits each event as a JSON object aligned with the OpenTelemetry log data model (severity_text, severity_number, body, attributes, resource, instrumentation_scope, trace_id/span_id from MDC). The example uses log4j-layout-template-json, which must be added to the classpath alongside log4j-core to enable JsonTemplateLayout. --- bookkeeper-benchmark/conf/log4j2.xml | 6 +- .../src/main/resources/log4j2.xml | 2 +- conf/log4j2.cli.xml | 2 +- conf/log4j2.otel-json.xml | 86 +++++++++++++++++++ conf/log4j2.shell.xml | 2 +- conf/log4j2.xml | 23 ++--- conf/otel-log-template.json | 37 ++++++++ stream/conf/log4j2.cli.xml | 6 +- stream/conf/log4j2.xml | 6 +- 9 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 conf/log4j2.otel-json.xml create mode 100644 conf/otel-log-template.json diff --git a/bookkeeper-benchmark/conf/log4j2.xml b/bookkeeper-benchmark/conf/log4j2.xml index 21257d6dc3a..67fa949756e 100644 --- a/bookkeeper-benchmark/conf/log4j2.xml +++ b/bookkeeper-benchmark/conf/log4j2.xml @@ -22,13 +22,13 @@ - + - + - + diff --git a/bookkeeper-server/src/main/resources/log4j2.xml b/bookkeeper-server/src/main/resources/log4j2.xml index 3e3e4bc8c8e..e604253877f 100644 --- a/bookkeeper-server/src/main/resources/log4j2.xml +++ b/bookkeeper-server/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/conf/log4j2.cli.xml b/conf/log4j2.cli.xml index 3a1d2fa4ffb..386927bd680 100644 --- a/conf/log4j2.cli.xml +++ b/conf/log4j2.cli.xml @@ -34,7 +34,7 @@ - + diff --git a/conf/log4j2.otel-json.xml b/conf/log4j2.otel-json.xml new file mode 100644 index 00000000000..f0b9ed3f30c --- /dev/null +++ b/conf/log4j2.otel-json.xml @@ -0,0 +1,86 @@ + + + + + + . + bookkeeper-server.json.log + INFO + CONSOLE + file:${sys:bookkeeper.conf.dir:-conf}/otel-log-template.json + ${env:OTEL_SERVICE_NAME:-bookkeeper} + ${env:HOSTNAME:-unknown} + + + + + + + + + + + + + + + + + + + + diff --git a/conf/log4j2.shell.xml b/conf/log4j2.shell.xml index c6e6b0fb9cd..1ed08d08ca3 100644 --- a/conf/log4j2.shell.xml +++ b/conf/log4j2.shell.xml @@ -31,7 +31,7 @@ - + diff --git a/conf/log4j2.xml b/conf/log4j2.xml index 08d5d5613c1..b0830eb82bc 100644 --- a/conf/log4j2.xml +++ b/conf/log4j2.xml @@ -27,26 +27,20 @@ CONSOLE + - - - - - - - - - - - - - @@ -58,8 +52,5 @@ - - - diff --git a/conf/otel-log-template.json b/conf/otel-log-template.json new file mode 100644 index 00000000000..298411dac5d --- /dev/null +++ b/conf/otel-log-template.json @@ -0,0 +1,37 @@ +{ + "timestamp": { + "$resolver": "timestamp", + "pattern": {"format": "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "timeZone": "UTC"} + }, + "observed_timestamp": { + "$resolver": "timestamp", + "pattern": {"format": "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "timeZone": "UTC"} + }, + "severity_text": {"$resolver": "level", "field": "name"}, + "severity_number": { + "$resolver": "pattern", + "pattern": "%level{TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21}" + }, + "body": {"$resolver": "message", "stringified": true}, + "trace_id": {"$resolver": "mdc", "key": "trace_id"}, + "span_id": {"$resolver": "mdc", "key": "span_id"}, + "trace_flags": {"$resolver": "mdc", "key": "trace_flags"}, + "instrumentation_scope": { + "name": {"$resolver": "logger", "field": "name"} + }, + "resource": { + "service.name": "${otel.service.name}", + "host.name": "${otel.host.name}" + }, + "attributes": {"$resolver": "mdc", "flatten": true}, + "thread_name": {"$resolver": "thread", "field": "name"}, + "exception": { + "type": {"$resolver": "exception", "field": "className"}, + "message": {"$resolver": "exception", "field": "message"}, + "stack_trace": { + "$resolver": "exception", + "field": "stackTrace", + "stackTrace": {"stringified": true} + } + } +} diff --git a/stream/conf/log4j2.cli.xml b/stream/conf/log4j2.cli.xml index 8c625013923..8614db12431 100644 --- a/stream/conf/log4j2.cli.xml +++ b/stream/conf/log4j2.cli.xml @@ -28,13 +28,13 @@ - + - + - + diff --git a/stream/conf/log4j2.xml b/stream/conf/log4j2.xml index 006e4bf6975..d4b0ad33e5b 100644 --- a/stream/conf/log4j2.xml +++ b/stream/conf/log4j2.xml @@ -28,13 +28,13 @@ - + - + - + From 5851d563cdd2a7085f08f6a79106a9845eeb33dd Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 28 Apr 2026 17:15:38 -0700 Subject: [PATCH 2/3] log4j2: render context map after the message --- bookkeeper-benchmark/conf/log4j2.xml | 6 +++--- bookkeeper-server/src/main/resources/log4j2.xml | 2 +- conf/log4j2.cli.xml | 2 +- conf/log4j2.shell.xml | 2 +- conf/log4j2.xml | 6 +++--- stream/conf/log4j2.cli.xml | 6 +++--- stream/conf/log4j2.xml | 6 +++--- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bookkeeper-benchmark/conf/log4j2.xml b/bookkeeper-benchmark/conf/log4j2.xml index 67fa949756e..e89054596ec 100644 --- a/bookkeeper-benchmark/conf/log4j2.xml +++ b/bookkeeper-benchmark/conf/log4j2.xml @@ -22,13 +22,13 @@ - + - + - + diff --git a/bookkeeper-server/src/main/resources/log4j2.xml b/bookkeeper-server/src/main/resources/log4j2.xml index e604253877f..b67f9faa5dd 100644 --- a/bookkeeper-server/src/main/resources/log4j2.xml +++ b/bookkeeper-server/src/main/resources/log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/conf/log4j2.cli.xml b/conf/log4j2.cli.xml index 386927bd680..6f2b2bf1b73 100644 --- a/conf/log4j2.cli.xml +++ b/conf/log4j2.cli.xml @@ -34,7 +34,7 @@ - + diff --git a/conf/log4j2.shell.xml b/conf/log4j2.shell.xml index 1ed08d08ca3..563149697ac 100644 --- a/conf/log4j2.shell.xml +++ b/conf/log4j2.shell.xml @@ -31,7 +31,7 @@ - + diff --git a/conf/log4j2.xml b/conf/log4j2.xml index b0830eb82bc..8aed8f0dab8 100644 --- a/conf/log4j2.xml +++ b/conf/log4j2.xml @@ -35,13 +35,13 @@ specific format. --> - + - + - + diff --git a/stream/conf/log4j2.cli.xml b/stream/conf/log4j2.cli.xml index 8614db12431..e29f7331281 100644 --- a/stream/conf/log4j2.cli.xml +++ b/stream/conf/log4j2.cli.xml @@ -28,13 +28,13 @@ - + - + - + diff --git a/stream/conf/log4j2.xml b/stream/conf/log4j2.xml index d4b0ad33e5b..fd16c99e490 100644 --- a/stream/conf/log4j2.xml +++ b/stream/conf/log4j2.xml @@ -28,13 +28,13 @@ - + - + - + From ddbda497275772900939230b78fb156473704048 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 28 Apr 2026 17:23:26 -0700 Subject: [PATCH 3/3] Align OTel JSON template with the OpenTelemetry log data model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the previous template with the structure used by Pulsar (apache/pulsar/blob/master/conf/OtelLogLayout.json): * Top-level fields use the spec's PascalCase naming (Timestamp, SeverityText, Body, TraceId, SpanId, TraceFlags, Attributes). * thread.name, thread.id, code.namespace, exception.{type,message, stacktrace}, and the flattened MDC are placed under "Attributes", using OTel semantic-convention keys. * Drop SeverityNumber, ObservedTimestamp, InstrumentationScope, and Resource — the first two are derivable / redundant for app-emitted logs, the logger name is "code.namespace" not InstrumentationScope, and Resource is normally attached by the collector pipeline. * The MDC resolver with flatten=true now sits inside Attributes, so slog .attr(...) keys merge as siblings of thread.name etc. (the previous placement at the top level didn't merge into "attributes" as intended). Update the explanatory comment block and drop the now-unused otel.service.name / otel.host.name properties. --- conf/log4j2.otel-json.xml | 34 ++++++++--------- conf/otel-log-template.json | 76 +++++++++++++++++++++++++------------ 2 files changed, 68 insertions(+), 42 deletions(-) diff --git a/conf/log4j2.otel-json.xml b/conf/log4j2.otel-json.xml index f0b9ed3f30c..a6df4c47358 100644 --- a/conf/log4j2.otel-json.xml +++ b/conf/log4j2.otel-json.xml @@ -24,9 +24,6 @@ object aligned with the OpenTelemetry log data model (https://opentelemetry.io/docs/specs/otel/logs/data-model/). - The structured attributes set via slog ".attr(...)" calls are written to - Log4j2's ContextData and surface here under the "attributes" field. - Required runtime dependency (NOT included by default in BookKeeper): org.apache.logging.log4j:log4j-layout-template-json:${log4j.version} @@ -36,18 +33,23 @@ there (e.g. add resource attributes, drop fields, rename keys) without touching this file. - Notes on the OTel mapping: - * "severity_text" = log4j level name ("INFO", "WARN", ...). - * "severity_number" = OTel severity number derived from the level - (TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17). - * "body" = the log message string. - * "trace_id"/"span_id" = MDC keys conventionally set by the OpenTelemetry - log appender bridge; omitted when absent. - * "attributes" = the full ContextData map (slog attrs + any - other entries placed in Log4j2 ThreadContext - or SLF4J MDC). - * "resource" = static attributes about the producer; populate - via OTEL_SERVICE_NAME / HOSTNAME env vars. + Top-level fields follow the OTel data-model field naming (PascalCase): + * "Timestamp" = event time, ISO-8601 UTC. + * "SeverityText" = log4j level name ("INFO", "WARN", ...). + * "Body" = the log message string. + * "TraceId" / "SpanId" / "TraceFlags" + = MDC keys conventionally set by the + OpenTelemetry log appender bridge; omitted + when absent. + * "Attributes" = OTel-semantic-conventions attributes for the + event: + - thread.name / thread.id + - code.namespace (= the logger name) + - exception.type / .message / .stacktrace + Plus the full Log4j2 ContextData map (every + slog ".attr(...)" entry and any SLF4J MDC + key) merged in via the "mdc" resolver with + flatten=true. --> @@ -56,8 +58,6 @@ INFO CONSOLE file:${sys:bookkeeper.conf.dir:-conf}/otel-log-template.json - ${env:OTEL_SERVICE_NAME:-bookkeeper} - ${env:HOSTNAME:-unknown} diff --git a/conf/otel-log-template.json b/conf/otel-log-template.json index 298411dac5d..d3e8e95f856 100644 --- a/conf/otel-log-template.json +++ b/conf/otel-log-template.json @@ -1,37 +1,63 @@ { - "timestamp": { + "Timestamp": { "$resolver": "timestamp", - "pattern": {"format": "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "timeZone": "UTC"} + "pattern": { + "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "timeZone": "UTC" + } }, - "observed_timestamp": { - "$resolver": "timestamp", - "pattern": {"format": "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", "timeZone": "UTC"} + "SeverityText": { + "$resolver": "level", + "field": "name" + }, + "Body": { + "$resolver": "message", + "stringified": true }, - "severity_text": {"$resolver": "level", "field": "name"}, - "severity_number": { - "$resolver": "pattern", - "pattern": "%level{TRACE=1, DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21}" + "TraceId": { + "$resolver": "mdc", + "key": "trace_id" }, - "body": {"$resolver": "message", "stringified": true}, - "trace_id": {"$resolver": "mdc", "key": "trace_id"}, - "span_id": {"$resolver": "mdc", "key": "span_id"}, - "trace_flags": {"$resolver": "mdc", "key": "trace_flags"}, - "instrumentation_scope": { - "name": {"$resolver": "logger", "field": "name"} + "SpanId": { + "$resolver": "mdc", + "key": "span_id" }, - "resource": { - "service.name": "${otel.service.name}", - "host.name": "${otel.host.name}" + "TraceFlags": { + "$resolver": "mdc", + "key": "trace_flags" }, - "attributes": {"$resolver": "mdc", "flatten": true}, - "thread_name": {"$resolver": "thread", "field": "name"}, - "exception": { - "type": {"$resolver": "exception", "field": "className"}, - "message": {"$resolver": "exception", "field": "message"}, - "stack_trace": { + "Attributes": { + "thread.name": { + "$resolver": "thread", + "field": "name" + }, + "thread.id": { + "$resolver": "thread", + "field": "id" + }, + "code.namespace": { + "$resolver": "logger", + "field": "name" + }, + "exception.type": { + "$resolver": "exception", + "field": "className" + }, + "exception.message": { + "$resolver": "exception", + "field": "message" + }, + "exception.stacktrace": { "$resolver": "exception", "field": "stackTrace", - "stackTrace": {"stringified": true} + "stackTrace": { + "stringified": true + } + }, + "mdc": { + "$resolver": "mdc", + "flatten": true, + "stringified": true } } }