This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 0c9f50382f [Cherry-pick to branch-1.3] [#11171] improvement(core): add
JSON formatter for audit logs (#11343) (#11482)
0c9f50382f is described below
commit 0c9f50382f18ed735f4083bfd4a601826b176852
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jun 9 08:17:14 2026 +0800
[Cherry-pick to branch-1.3] [#11171] improvement(core): add JSON formatter
for audit logs (#11343) (#11482)
**Cherry-pick Information:**
- Original commit: 6932ccaca1f17c83ecacf6d4e98282e86f5bbe14
- Target branch: `branch-1.3`
- Status: ✅ Clean cherry-pick (no conflicts)
Co-authored-by: Shane <[email protected]>
Co-authored-by: Qi Yu <[email protected]>
---
.../apache/gravitino/audit/AuditLogRedactor.java | 70 +++++++++
.../apache/gravitino/audit/JsonAuditFormatter.java | 84 +++++++++++
.../gravitino/audit/v2/SimpleAuditLogV2.java | 3 +-
.../gravitino/audit/TestJsonAuditFormatter.java | 168 +++++++++++++++++++++
.../gravitino/resources/config/gravitino.conf | 4 +-
docs/gravitino-server-config.md | 2 +-
6 files changed, 327 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/audit/AuditLogRedactor.java
b/core/src/main/java/org/apache/gravitino/audit/AuditLogRedactor.java
new file mode 100644
index 0000000000..a60dece5d4
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/audit/AuditLogRedactor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.audit;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Utility methods for redacting sensitive values in audit logs. */
+public final class AuditLogRedactor {
+
+ /** Redacted value used for sensitive audit fields. */
+ public static final String REDACTED_VALUE = "***";
+
+ private static final Set<String> MASKED_CUSTOM_INFO_KEYS =
+ ImmutableSet.of(
+ "authorization", "cookie", "x-amz-security-token",
"s3.access-key-id", "jdbc-password");
+
+ private AuditLogRedactor() {}
+
+ /**
+ * Redacts sensitive custom information values while preserving the original
key order.
+ *
+ * @param customInfo the original custom information
+ * @return a copy with sensitive values redacted
+ */
+ public static Map<String, String> redactCustomInfo(Map<String, String>
customInfo) {
+ Map<String, String> redacted = new LinkedHashMap<>();
+ if (customInfo == null) {
+ return redacted;
+ }
+
+ customInfo.forEach((key, value) -> redacted.put(key, redactValue(key,
value)));
+ return redacted;
+ }
+
+ /**
+ * Redacts a value when its key is considered sensitive.
+ *
+ * @param key the custom information key
+ * @param value the original value
+ * @return the redacted value for sensitive keys, otherwise the original
value
+ */
+ public static String redactValue(String key, String value) {
+ return isSensitiveCustomInfoKey(key) ? REDACTED_VALUE : value;
+ }
+
+ private static boolean isSensitiveCustomInfoKey(String key) {
+ return key != null &&
MASKED_CUSTOM_INFO_KEYS.contains(key.toLowerCase(Locale.ROOT));
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/audit/JsonAuditFormatter.java
b/core/src/main/java/org/apache/gravitino/audit/JsonAuditFormatter.java
new file mode 100644
index 0000000000..b048aefdcd
--- /dev/null
+++ b/core/src/main/java/org/apache/gravitino/audit/JsonAuditFormatter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.audit;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.gravitino.audit.v2.SimpleAuditLogV2;
+import org.apache.gravitino.listener.api.event.Event;
+import org.apache.gravitino.listener.api.event.ListEvent;
+
+/** Formatter that serializes audit logs as one JSON object per line. */
+public class JsonAuditFormatter implements Formatter {
+
+ @Override
+ public AuditLog format(Event event) {
+ return new JsonAuditLog(event);
+ }
+}
+
+final class JsonAuditLog extends SimpleAuditLogV2 {
+
+ private static final ObjectMapper OBJECT_MAPPER =
JsonMapper.builder().build();
+
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER =
+
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneId.systemDefault());
+
+ private final Event event;
+
+ JsonAuditLog(Event event) {
+ super(event);
+ this.event = event;
+ }
+
+ @Override
+ public String toString() {
+ Map<String, Object> payload = new LinkedHashMap<>();
+ payload.put("timestamp",
TIMESTAMP_FORMATTER.format(Instant.ofEpochMilli(timestamp())));
+ payload.put("user", user());
+ payload.put("operation", operation());
+ payload.put("operationType", operationType());
+ payload.put("identifier", identifier());
+ payload.put("status", status());
+ payload.put("operationStatus", operationStatus());
+ payload.put("eventSource", eventSource());
+ payload.put("remoteAddress", remoteAddress());
+ payload.put("customInfo", AuditLogRedactor.redactCustomInfo(customInfo()));
+
+ if (event instanceof ListEvent) {
+ int resultCount = ((ListEvent) event).resultCount();
+ if (resultCount >= 0) {
+ payload.put("resultCount", resultCount);
+ }
+ }
+
+ try {
+ return OBJECT_MAPPER.writeValueAsString(payload);
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException("Failed to serialize audit log to JSON",
e);
+ }
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/audit/v2/SimpleAuditLogV2.java
b/core/src/main/java/org/apache/gravitino/audit/v2/SimpleAuditLogV2.java
index 99a6ecb012..cd49246e62 100644
--- a/core/src/main/java/org/apache/gravitino/audit/v2/SimpleAuditLogV2.java
+++ b/core/src/main/java/org/apache/gravitino/audit/v2/SimpleAuditLogV2.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Optional;
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.audit.AuditLog;
+import org.apache.gravitino.audit.AuditLogRedactor;
import org.apache.gravitino.listener.api.event.BaseEvent;
import org.apache.gravitino.listener.api.event.EventSource;
import org.apache.gravitino.listener.api.event.ListEvent;
@@ -106,7 +107,7 @@ public class SimpleAuditLogV2 implements AuditLog {
Map<String, String> info = customInfo();
List<String> parts = new ArrayList<>();
if (info != null) {
- info.forEach((k, v) -> parts.add(k + "=" + v));
+ info.forEach((k, v) -> parts.add(k + "=" +
AuditLogRedactor.redactValue(k, v)));
}
if (event instanceof ListEvent) {
int count = ((ListEvent) event).resultCount();
diff --git
a/core/src/test/java/org/apache/gravitino/audit/TestJsonAuditFormatter.java
b/core/src/test/java/org/apache/gravitino/audit/TestJsonAuditFormatter.java
new file mode 100644
index 0000000000..a22e953dfd
--- /dev/null
+++ b/core/src/test/java/org/apache/gravitino/audit/TestJsonAuditFormatter.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.audit;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import java.time.OffsetDateTime;
+import java.util.Map;
+import org.apache.gravitino.NameIdentifier;
+import org.apache.gravitino.audit.v2.SimpleFormatterV2;
+import org.apache.gravitino.json.JsonUtils;
+import org.apache.gravitino.listener.api.event.Event;
+import org.apache.gravitino.listener.api.event.EventSource;
+import org.apache.gravitino.listener.api.event.ListEvent;
+import org.apache.gravitino.listener.api.event.ListMetalakeEvent;
+import org.apache.gravitino.listener.api.event.OperationStatus;
+import org.apache.gravitino.listener.api.event.OperationType;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestJsonAuditFormatter {
+
+ private final JsonAuditFormatter formatter = new JsonAuditFormatter();
+
+ @Test
+ public void testFormatSerializesAllCoreFields() throws Exception {
+ JsonNode node = toJsonNode(formatter.format(new StubEvent()));
+
+ Assertions.assertEquals("test-user", node.get("user").asText());
+ Assertions.assertEquals("LIST_TABLE", node.get("operation").asText());
+ Assertions.assertEquals("LIST_TABLE", node.get("operationType").asText());
+ Assertions.assertEquals("metalake.catalog",
node.get("identifier").asText());
+ Assertions.assertEquals("SUCCESS", node.get("status").asText());
+ Assertions.assertEquals("SUCCESS", node.get("operationStatus").asText());
+ Assertions.assertEquals("GRAVITINO_SERVER",
node.get("eventSource").asText());
+ Assertions.assertEquals("unknown", node.get("remoteAddress").asText());
+ Assertions.assertTrue(node.has("customInfo"));
+ Assertions.assertTrue(node.get("customInfo").isObject());
+ Assertions.assertEquals(0, node.get("customInfo").size());
+
+ OffsetDateTime timestamp =
OffsetDateTime.parse(node.get("timestamp").asText());
+ Assertions.assertEquals(0, timestamp.getNano() % 1_000_000);
+ }
+
+ @Test
+ public void testSensitiveCustomInfoIsMasked() throws Exception {
+ JsonNode node = toJsonNode(formatter.format(new
StubEventWithSensitiveCustomInfo()));
+ JsonNode customInfo = node.get("customInfo");
+
+ Assertions.assertEquals("***", customInfo.get("Authorization").asText());
+ Assertions.assertEquals("***", customInfo.get("cookie").asText());
+ Assertions.assertEquals("***",
customInfo.get("X-Amz-Security-Token").asText());
+ Assertions.assertEquals("***",
customInfo.get("s3.access-key-id").asText());
+ Assertions.assertEquals("***", customInfo.get("jdbc-password").asText());
+ Assertions.assertEquals("visible", customInfo.get("env").asText());
+ }
+
+ @Test
+ public void testSimpleFormatterV2MasksSensitiveCustomInfo() {
+ String formatted =
+ new SimpleFormatterV2().format(new
StubEventWithSensitiveCustomInfo()).toString();
+
+ Assertions.assertTrue(formatted.contains("Authorization=***"));
+ Assertions.assertTrue(formatted.contains("cookie=***"));
+ Assertions.assertTrue(formatted.contains("X-Amz-Security-Token=***"));
+ Assertions.assertTrue(formatted.contains("s3.access-key-id=***"));
+ Assertions.assertTrue(formatted.contains("jdbc-password=***"));
+ Assertions.assertTrue(formatted.contains("env=visible"));
+ Assertions.assertFalse(formatted.contains("Bearer token"));
+ Assertions.assertFalse(formatted.contains("session-token"));
+ Assertions.assertFalse(formatted.contains("secret"));
+ }
+
+ @Test
+ public void testListEventAddsResultCount() throws Exception {
+ StubListEvent event = new StubListEvent();
+
+ JsonNode node = toJsonNode(formatter.format(event));
+ Assertions.assertEquals("LIST_TABLE", node.get("operationType").asText());
+ Assertions.assertEquals(4, node.get("resultCount").asInt());
+ Assertions.assertEquals("prod",
node.get("customInfo").get("env").asText());
+ }
+
+ @Test
+ public void testNullIdentifierSerializedAsNull() throws Exception {
+ JsonNode node = toJsonNode(formatter.format(new ListMetalakeEvent("bob",
2)));
+
+ Assertions.assertTrue(node.has("identifier"));
+ Assertions.assertTrue(node.get("identifier").isNull());
+ Assertions.assertEquals(2, node.get("resultCount").asInt());
+ }
+
+ private JsonNode toJsonNode(AuditLog auditLog) throws Exception {
+ return JsonUtils.objectMapper().readTree(auditLog.toString());
+ }
+
+ static class StubEvent extends Event {
+ StubEvent() {
+ this("test-user", NameIdentifier.of("metalake", "catalog"));
+ }
+
+ StubEvent(String user, NameIdentifier identifier) {
+ super(user, identifier);
+ }
+
+ @Override
+ public OperationType operationType() {
+ return OperationType.LIST_TABLE;
+ }
+
+ @Override
+ public OperationStatus operationStatus() {
+ return OperationStatus.SUCCESS;
+ }
+
+ @Override
+ public EventSource eventSource() {
+ return EventSource.GRAVITINO_SERVER;
+ }
+ }
+
+ static class StubEventWithSensitiveCustomInfo extends StubEvent {
+ @Override
+ public Map<String, String> customInfo() {
+ return ImmutableMap.<String, String>builder()
+ .put("Authorization", "Bearer token")
+ .put("cookie", "a=b")
+ .put("X-Amz-Security-Token", "session-token")
+ .put("s3.access-key-id", "ak")
+ .put("jdbc-password", "secret")
+ .put("env", "visible")
+ .build();
+ }
+ }
+
+ static class StubListEvent extends StubEvent implements ListEvent {
+ StubListEvent() {
+ super("alice", NameIdentifier.of("m", "c", "s"));
+ }
+
+ @Override
+ public int resultCount() {
+ return 4;
+ }
+
+ @Override
+ public Map<String, String> customInfo() {
+ return ImmutableMap.of("env", "prod");
+ }
+ }
+}
diff --git a/dev/charts/gravitino/resources/config/gravitino.conf
b/dev/charts/gravitino/resources/config/gravitino.conf
index 1d3f20f33c..5855b37d95 100644
--- a/dev/charts/gravitino/resources/config/gravitino.conf
+++ b/dev/charts/gravitino/resources/config/gravitino.conf
@@ -173,7 +173,7 @@ gravitino.iceberg-rest.default-catalog-name = {{
.Values.icebergRest.dynamicConf
# Audit log configuration
gravitino.audit.enabled = {{ .Values.audit.enabled }}
gravitino.audit.writer.className = {{ if (and .Values.audit
.Values.audit.writer .Values.audit.writer.className) }}{{
.Values.audit.writer.className }}{{ else
}}org.apache.gravitino.audit.FileAuditWriter{{- end }}
-gravitino.audit.formatter.className = {{ if (and .Values.audit
.Values.audit.formatter .Values.audit.formatter.className) }}{{
.Values.audit.formatter.className }}{{ else
}}org.apache.gravitino.audit.SimpleFormatter{{- end }}
+gravitino.audit.formatter.className = {{ if (and .Values.audit
.Values.audit.formatter .Values.audit.formatter.className) }}{{
.Values.audit.formatter.className }}{{ else
}}org.apache.gravitino.audit.v2.SimpleFormatterV2{{- end }}
gravitino.audit.writer.file.fileName = {{ .Values.audit.writer.file.fileName }}
gravitino.audit.writer.file.flushIntervalSecs = {{
.Values.audit.writer.file.flushIntervalSecs }}
gravitino.audit.writer.file.append = {{ .Values.audit.writer.file.append }}
@@ -189,4 +189,4 @@ gravitino.server.visibleConfigs = {{ .Values.visibleConfigs
}}
{{- end }}
{{- range $key, $val := .Values.additionalConfigItems }}
{{ $key }} = {{ tpl $val $ }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
index b5c8bfa708..c47e0c85f3 100644
--- a/docs/gravitino-server-config.md
+++ b/docs/gravitino-server-config.md
@@ -259,7 +259,7 @@ Gravitino provides a default implementation to log basic
audit information to a
#### Audit Log Formatter
-The `Formatter` interface transforms an `Event` into an `AuditLog`.
`SimpleFormatterV2` is the default implementation and requires no extra
configuration. It produces a tab-separated line with the following fields:
timestamp, user, operation type, identifier, operation status, event source,
remote address, and custom info.
+The `Formatter` interface transforms an `Event` into an `AuditLog`.
`SimpleFormatterV2` is the default implementation. `JsonAuditFormatter` is also
available when structured JSON output is required. It emits one JSON object per
line, serializes `customInfo`, and formats `timestamp` as ISO 8601 with
millisecond precision and zone offset. Both simple and JSON formatters mask
sensitive values such as `Authorization`, `Cookie`, `X-Amz-Security-Token`,
`s3.access-key-id`, and `jdbc-password`.
#### Audit Log Writer
