This is an automated email from the ASF dual-hosted git repository.
capistrant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c6ef2fedb58 Intern VirtualColumns containers where possible to help
reduce memory overhead in a few places (#20563)
c6ef2fedb58 is described below
commit c6ef2fedb5844334f9aed7e64116880b61b04c08
Author: Lucas Capistrant <[email protected]>
AuthorDate: Fri Oct 16 08:00:46 2026 -0500
Intern VirtualColumns containers where possible to help reduce memory
overhead in a few places (#20563)
---
.../apache/druid/timeline/ClusterGroupTuples.java | 16 ++-------
.../org/apache/druid/timeline/DataSegment.java | 27 +++++++++++++--
.../partition/BaseDimensionRangeShardSpec.java | 6 +---
.../druid/timeline/ClusterGroupTuplesTest.java | 1 +
.../partition/DimensionRangeShardSpecTest.java | 39 ++++++++++++++++++++++
5 files changed, 68 insertions(+), 21 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/timeline/ClusterGroupTuples.java
b/processing/src/main/java/org/apache/druid/timeline/ClusterGroupTuples.java
index 7b16295d3a1..d56c54ec0f4 100644
--- a/processing/src/main/java/org/apache/druid/timeline/ClusterGroupTuples.java
+++ b/processing/src/main/java/org/apache/druid/timeline/ClusterGroupTuples.java
@@ -43,7 +43,7 @@ import java.util.Objects;
* segment pruning can make use of this information.
* <p>
* The compact constructor validates {@code clusteringColumns}, interns the
virtual columns through
- * {@link DataSegment#virtualColumnInterner()}, and canonicalizes every tuple
value to its declared
+ * {@link DataSegment#internVirtualColumns}, and canonicalizes every tuple
value to its declared
* {@link ColumnType} via {@link #coerceValue} so {@link Object#equals} works
across the JSON/programmatic boundary.
*/
public record ClusterGroupTuples(
@@ -58,7 +58,7 @@ public record ClusterGroupTuples(
if (clusteringColumns == null || clusteringColumns.size() == 0) {
throw InvalidInput.exception("clusteringColumns must not be null or
empty");
}
- virtualColumns = internVirtualColumns(virtualColumns);
+ virtualColumns = DataSegment.internVirtualColumns(virtualColumns);
tuples = canonicalizeTuples(clusteringColumns, tuples);
}
@@ -125,18 +125,6 @@ public record ClusterGroupTuples(
);
}
- private static VirtualColumns internVirtualColumns(@Nullable VirtualColumns
virtualColumns)
- {
- if (virtualColumns == null || virtualColumns.isEmpty()) {
- return VirtualColumns.EMPTY;
- }
- return VirtualColumns.create(
- Arrays.stream(virtualColumns.getVirtualColumns())
- .map(DataSegment.virtualColumnInterner()::intern)
- .toList()
- );
- }
-
private static List<List<Object>> canonicalizeTuples(
RowSignature clusteringColumns,
@Nullable List<List<Object>> tuples
diff --git
a/processing/src/main/java/org/apache/druid/timeline/DataSegment.java
b/processing/src/main/java/org/apache/druid/timeline/DataSegment.java
index 9e9208bfcd2..0172f7acd07 100644
--- a/processing/src/main/java/org/apache/druid/timeline/DataSegment.java
+++ b/processing/src/main/java/org/apache/druid/timeline/DataSegment.java
@@ -40,11 +40,13 @@ import org.apache.druid.jackson.CommaListJoinSerializer;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.query.SegmentDescriptor;
import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.timeline.partition.NumberedShardSpec;
import org.apache.druid.timeline.partition.ShardSpec;
import org.joda.time.Interval;
import javax.annotation.Nullable;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -68,17 +70,37 @@ public class DataSegment implements
Comparable<DataSegment>, Overshadowable<Data
}
/**
- * Shared canonical interner for {@link
org.apache.druid.segment.VirtualColumn} instances embedded in segment-side
+ * Shared canonical interner for {@link VirtualColumn} instances embedded in
segment-side
* metadata that lives in broker/coordinator memory. Used by callers that
carry virtual columns through
* {@link DataSegment}'s wire form (clustering virtual columns on {@link
ClusterGroupTuples},
* {@link org.apache.druid.timeline.partition.BaseDimensionRangeShardSpec}'s
range-clustering virtual columns) so
- * identical VC definitions across segments collapse to a single instance in
memory.
+ * identical VC definitions across segments collapse to a single instance in
memory. Callers holding a whole
+ * {@link VirtualColumns} should use {@link #internVirtualColumns}.
*/
public static Interner<VirtualColumn> virtualColumnInterner()
{
return VIRTUAL_COLUMN_INTERNER;
}
+ /**
+ * Returns a shared {@link VirtualColumns} instance equal to the given one,
so segments with the same virtual column
+ * definitions reuse one object in memory instead of each holding its own
copy. Returns {@link VirtualColumns#EMPTY}
+ * if the input is null or empty.
+ */
+ public static VirtualColumns internVirtualColumns(@Nullable VirtualColumns
virtualColumns)
+ {
+ if (virtualColumns == null || virtualColumns.isEmpty()) {
+ return VirtualColumns.EMPTY;
+ }
+ return VIRTUAL_COLUMNS_INTERNER.intern(
+ VirtualColumns.create(
+ Arrays.stream(virtualColumns.getVirtualColumns())
+ .map(VIRTUAL_COLUMN_INTERNER::intern)
+ .toList()
+ )
+ );
+ }
+
/*
* The difference between this class and org.apache.druid.segment.Segment is
that this class contains the segment
* metadata only, while org.apache.druid.segment.Segment represents the
actual body of segment data, queryable.
@@ -109,6 +131,7 @@ public class DataSegment implements
Comparable<DataSegment>, Overshadowable<Data
private static final Interner<List<String>> PROJECTIONS_INTERNER =
Interners.newWeakInterner();
private static final Interner<ClusterGroupTuples> CLUSTER_GROUPS_INTERNER =
Interners.newWeakInterner();
private static final Interner<VirtualColumn> VIRTUAL_COLUMN_INTERNER =
Interners.newWeakInterner();
+ private static final Interner<VirtualColumns> VIRTUAL_COLUMNS_INTERNER =
Interners.newWeakInterner();
private static final Interner<CompactionState> COMPACTION_STATE_INTERNER =
Interners.newWeakInterner();
private static final Map<String, Object> PRUNED_LOAD_SPEC = ImmutableMap.of(
"load spec is pruned, because it's not needed on Brokers, but eats a lot
of heap space",
diff --git
a/processing/src/main/java/org/apache/druid/timeline/partition/BaseDimensionRangeShardSpec.java
b/processing/src/main/java/org/apache/druid/timeline/partition/BaseDimensionRangeShardSpec.java
index 36a761a0955..a8e103e335d 100644
---
a/processing/src/main/java/org/apache/druid/timeline/partition/BaseDimensionRangeShardSpec.java
+++
b/processing/src/main/java/org/apache/druid/timeline/partition/BaseDimensionRangeShardSpec.java
@@ -49,11 +49,7 @@ public abstract class BaseDimensionRangeShardSpec implements
ShardSpec
)
{
this.dimensions =
dimensions.stream().map(DataSegment.stringInterner()::intern).toList();
- this.virtualColumns = virtualColumns == null
- ? VirtualColumns.EMPTY
- :
VirtualColumns.create(Arrays.stream(virtualColumns.getVirtualColumns())
-
.map(DataSegment.virtualColumnInterner()::intern)
- .toList());
+ this.virtualColumns = DataSegment.internVirtualColumns(virtualColumns);
this.start = start;
this.end = end;
}
diff --git
a/processing/src/test/java/org/apache/druid/timeline/ClusterGroupTuplesTest.java
b/processing/src/test/java/org/apache/druid/timeline/ClusterGroupTuplesTest.java
index a6a8c400c9f..e3be1f75743 100644
---
a/processing/src/test/java/org/apache/druid/timeline/ClusterGroupTuplesTest.java
+++
b/processing/src/test/java/org/apache/druid/timeline/ClusterGroupTuplesTest.java
@@ -351,6 +351,7 @@ class ClusterGroupTuplesTest
VIRTUAL_COLUMNS,
List.of(List.of("globex"))
);
+ Assertions.assertSame(a.virtualColumns(), b.virtualColumns());
Assertions.assertSame(
a.virtualColumns().getVirtualColumns()[0],
b.virtualColumns().getVirtualColumns()[0]
diff --git
a/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
b/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
index 16c2866c329..c33fd74bd78 100644
---
a/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
+++
b/processing/src/test/java/org/apache/druid/timeline/partition/DimensionRangeShardSpecTest.java
@@ -27,7 +27,10 @@ import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.MapBasedInputRow;
import org.apache.druid.data.input.StringTuple;
import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -542,6 +545,42 @@ public class DimensionRangeShardSpecTest
assertTrue(shard.possibleInDomain(domain));
}
+ @Test
+ public void testEqualVirtualColumnsShareInstance()
+ {
+ setDimensions("v0");
+
+ final DimensionRangeShardSpec shard0 = new DimensionRangeShardSpec(
+ dimensions,
+ createVirtualColumns(),
+ null,
+ StringTuple.create("m"),
+ 0,
+ 2
+ );
+ final DimensionRangeShardSpec shard1 = new DimensionRangeShardSpec(
+ dimensions,
+ createVirtualColumns(),
+ StringTuple.create("m"),
+ null,
+ 1,
+ 2
+ );
+
+ Assertions.assertSame(shard0.getVirtualColumns(),
shard1.getVirtualColumns());
+ Assertions.assertSame(
+ shard0.getVirtualColumns().getVirtualColumns()[0],
+ shard1.getVirtualColumns().getVirtualColumns()[0]
+ );
+ }
+
+ private static VirtualColumns createVirtualColumns()
+ {
+ return VirtualColumns.create(
+ new ExpressionVirtualColumn("v0", "lower(\"dim\")", ColumnType.STRING,
TestExprMacroTable.INSTANCE)
+ );
+ }
+
private RangeSet<String> getRangeSet(Range range)
{
RangeSet<String> rangeSet = TreeRangeSet.create();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
