This is an automated email from the ASF dual-hosted git repository. abhishekrb pushed a commit to branch ordered_event_map in repository https://gitbox.apache.org/repos/asf/druid.git
commit 6e2ef69052fdbecd11b55849c3f1bd62a1a152ad Author: Abhishek Balaji Radhakrishnan <[email protected]> AuthorDate: Fri Feb 6 13:48:10 2026 -0800 Use a LinkedHashMap for EventMap so order of insertion is preserved. Currently, when we use LoggingEmitter and/or LoggingRequestLogger, long, complex SQL queries make up the majority of the log volume, so we truncate logs to some extent. With metric events, this often times lead to the "value" part among other getting truncated. So this patch switches the EventMap to a LinkedHashMap so critical info (e.g., metric name and value) are logged first, followed by fields like service dimensions, user dimensions and the SQL query itself. Same idea for the request logger: we log the query context before the SQL query itself, which lets us preserve any useful/debuggable information deterministically even under truncation. Given these maps typically have at most ~20–30 key/value pairs, the additional overhead of using a LinkedHashMap should be okay. --- .../druid/java/util/emitter/core/EventMap.java | 6 ++- .../util/emitter/service/ServiceMetricEvent.java | 2 +- .../druid/java/util/emitter/core/EventMapTest.java | 48 ++++++++++++++++++++++ .../emitter/service/ServiceMetricEventTest.java | 25 +++++++++++ .../org/apache/druid/server/RequestLogLine.java | 11 +++-- .../druid/server/log/DefaultRequestLogEvent.java | 5 +-- .../server/log/DefaultRequestLogEventTest.java | 13 +++++- 7 files changed, 98 insertions(+), 12 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/emitter/core/EventMap.java b/processing/src/main/java/org/apache/druid/java/util/emitter/core/EventMap.java index 50237eb0238..e1f5d8e13a3 100644 --- a/processing/src/main/java/org/apache/druid/java/util/emitter/core/EventMap.java +++ b/processing/src/main/java/org/apache/druid/java/util/emitter/core/EventMap.java @@ -22,16 +22,18 @@ package org.apache.druid.java.util.emitter.core; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; /** - * EventMap is a hash map implementation. It can be safely serialzed to JSON using Jackson serializer as it + * EventMap is a linked hash map implementation where the insertion order of key-value pairs is maintained. + * It can be safely serialzed to JSON using Jackson serializer as it * respects the polymorphic annotations on entires (unlike standard Map). The example of polymorphic class is a query * interface, where different native query types are resolved by additional field called "queryType". * This implementation ensures that the annotation on the values are respected during serialization. */ @JsonSerialize(using = EventMapSerializer.class) -public class EventMap extends HashMap<String, Object> +public class EventMap extends LinkedHashMap<String, Object> { /** * Returns builder with Fluent API to build EventMap instance using method chaining diff --git a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java index 41395ffea58..31f77a51283 100644 --- a/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java +++ b/processing/src/main/java/org/apache/druid/java/util/emitter/service/ServiceMetricEvent.java @@ -116,9 +116,9 @@ public class ServiceMetricEvent implements Event .builder() .put(FEED, getFeed()) .put(TIMESTAMP, createdTime.toString()) - .putAll(serviceDims) .put(METRIC, metric) .put(VALUE, value) + .putAll(serviceDims) .putAll(userDims) .build(); } diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/core/EventMapTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/core/EventMapTest.java new file mode 100644 index 00000000000..68a6b90a9af --- /dev/null +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/core/EventMapTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.java.util.emitter.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +public class EventMapTest +{ + @Test + public void testSerializedEventMapIsOrdered() throws JsonProcessingException + { + final EventMap map = EventMap.builder() + .put("k1", "v1") + .put("k2", "v2") + .put("k3", "v3") + .put("k4", "v4") + .put("k5", "v5") + .put("k0", "v0") + .put("a0", "a0") + .put("x0", "x0") + .build(); + + Assert.assertEquals( + "{\"k1\":\"v1\",\"k2\":\"v2\",\"k3\":\"v3\",\"k4\":\"v4\",\"k5\":\"v5\",\"k0\":\"v0\",\"a0\":\"a0\",\"x0\":\"x0\"}", + new DefaultObjectMapper().writeValueAsString(map) + ); + } +} \ No newline at end of file diff --git a/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java b/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java index 2b942870802..a9d58facf85 100644 --- a/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java +++ b/processing/src/test/java/org/apache/druid/java/util/emitter/service/ServiceMetricEventTest.java @@ -20,9 +20,14 @@ package org.apache.druid.java.util.emitter.service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.emitter.core.EventMap; +import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; @@ -51,6 +56,7 @@ public class ServiceMetricEventTest .setDimension("user10", "j") .setMetric("test-metric", 1234) .build("test", "localhost"); + Assert.assertEquals( ImmutableMap.<String, Object>builder() .put("feed", "metrics") @@ -359,4 +365,23 @@ public class ServiceMetricEventTest () -> eventBuilder.setDimension(null, new String[]{"a"}) ); } + + @Test + public void testSerializedServiceMetricEventIsOrdered() throws JsonProcessingException + { + ObjectMapper mapper = new DefaultObjectMapper(); + ServiceMetricEvent event = ServiceMetricEvent.builder() + .setCreatedTime(DateTimes.of("2026-01-01")) + .setMetric("m1", 42) + .setDimension("dim1", "xyz") + .setFeed("test_feed") + .setDimension("dim2", "xyz") + .build("broker", "hostA"); + + + Assert.assertEquals( + "{\"feed\":\"test_feed\",\"timestamp\":\"2026-01-01T00:00:00.000Z\",\"metric\":\"m1\",\"value\":42,\"service\":\"broker\",\"host\":\"hostA\",\"dim1\":\"xyz\",\"dim2\":\"xyz\"}", + mapper.writeValueAsString(event.toMap()) + ); + } } diff --git a/server/src/main/java/org/apache/druid/server/RequestLogLine.java b/server/src/main/java/org/apache/druid/server/RequestLogLine.java index 0e744520cc3..9d22f07c5f0 100644 --- a/server/src/main/java/org/apache/druid/server/RequestLogLine.java +++ b/server/src/main/java/org/apache/druid/server/RequestLogLine.java @@ -31,6 +31,7 @@ import org.joda.time.DateTime; import javax.annotation.Nullable; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -92,16 +93,18 @@ public class RequestLogLine public String getSqlQueryLine(ObjectMapper objectMapper) throws JsonProcessingException { + + final Map<String, Object> queryMap = new LinkedHashMap<>(); + queryMap.put("context", sqlQueryContext); + queryMap.put("query", sql == null ? "<unavailable>" : sql); + return JOINER.join( Arrays.asList( timestamp, remoteAddr, "", objectMapper.writeValueAsString(queryStats), - objectMapper.writeValueAsString(ImmutableMap.of( - "query", sql == null ? "<unavailable>" : sql, - "context", sqlQueryContext - )) + objectMapper.writeValueAsString(queryMap) ) ); } diff --git a/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java b/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java index d7f75d216eb..e22131f09f1 100644 --- a/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java +++ b/server/src/main/java/org/apache/druid/server/log/DefaultRequestLogEvent.java @@ -63,6 +63,8 @@ public final class DefaultRequestLogEvent implements RequestLogEvent .put("timestamp", getCreatedTime()) .put("service", getService()) .put("host", getHost()) + .put("remoteAddr", getRemoteAddr()) + .put("queryStats", getQueryStats()) .putNonNull("query", getQuery()); if (getSql() != null) { @@ -70,9 +72,6 @@ public final class DefaultRequestLogEvent implements RequestLogEvent .put("sqlQueryContext", getSqlQueryContext()); } - builder.put("remoteAddr", getRemoteAddr()) - .put("queryStats", getQueryStats()); - return builder.build(); } diff --git a/server/src/test/java/org/apache/druid/server/log/DefaultRequestLogEventTest.java b/server/src/test/java/org/apache/druid/server/log/DefaultRequestLogEventTest.java index 0f7e00b484e..631a4d4ccb0 100644 --- a/server/src/test/java/org/apache/druid/server/log/DefaultRequestLogEventTest.java +++ b/server/src/test/java/org/apache/druid/server/log/DefaultRequestLogEventTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.emitter.core.Event; import org.apache.druid.java.util.emitter.core.EventMap; @@ -126,7 +127,7 @@ public class DefaultRequestLogEventTest } @Test - public void testDefaultRequestLogEventToMapSQL() + public void testDefaultRequestLogEventToMapSQL() throws JsonProcessingException { final String feed = "test"; final DateTime timestamp = DateTimes.of(2019, 12, 12, 3, 1); @@ -164,7 +165,15 @@ public class DefaultRequestLogEventTest expected.put("remoteAddr", host); expected.put("queryStats", queryStats); - Assert.assertEquals(expected, defaultRequestLogEvent.toMap()); + EventMap observedMap = defaultRequestLogEvent.toMap(); + Assert.assertEquals(expected, observedMap); + Assert.assertEquals( + StringUtils.format( + "{\"feed\":\"test\",\"timestamp\":\"%s\",\"service\":\"druid-service\",\"host\":\"127.0.0.1\",\"remoteAddr\":\"127.0.0.1\",\"queryStats\":{\"sqlQuery/time\":13,\"sqlQuery/planningTimeMs\":1,\"sqlQuery/bytes\":10,\"success\":true,\"identity\":\"allowAll\"},\"sql\":\"select * from 1337\",\"sqlQueryContext\":{}}", + timestamp + ), + new DefaultObjectMapper().writeValueAsString(observedMap) + ); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
