This is an automated email from the ASF dual-hosted git repository.
clintropolis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new f0f69ff5218 minor: add fileGroup to V10 segment
SegmentFileContainerMetadata, switch default metadata compression to zstd
(#19468)
f0f69ff5218 is described below
commit f0f69ff521804c9edab9f0a743bfefd9adbcfe46
Author: Clint Wylie <[email protected]>
AuthorDate: Mon May 18 09:56:39 2026 -0700
minor: add fileGroup to V10 segment SegmentFileContainerMetadata, switch
default metadata compression to zstd (#19468)
---
.../embedded/query/QueryVirtualStorageTest.java | 16 +-
.../java/org/apache/druid/segment/IndexSpec.java | 3 +-
.../druid/segment/file/SegmentFileBuilderV10.java | 30 +++-
.../segment/file/SegmentFileContainerMetadata.java | 56 ++++++-
.../segment/file/SegmentFileBuilderV10Test.java | 177 +++++++++++++++++++--
.../file/SegmentFileContainerMetadataTest.java | 68 ++++++++
.../server/compaction/CompactionStatusTest.java | 4 +-
7 files changed, 327 insertions(+), 27 deletions(-)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
index 0b512537dec..df55055751d 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
@@ -66,8 +66,9 @@ import java.util.concurrent.ThreadLocalRandom;
*/
class QueryVirtualStorageTest extends EmbeddedClusterTestBase
{
- // size of wiki segments, adjust this if segment size changes for some reason
- private static final long SIZE_BYTES = 3777834L;
+ // size of wiki segments (size here is size with uncompressed metadata as an
upper bound since the zstd default
+ // appears to make different sizes on different platforms) adjust this if
segment size changes for some reason
+ private static final long SIZE_BYTES = 3778338L;
private static final long CACHE_SIZE = HumanReadableBytes.parse("1MiB");
private static final long MAX_SIZE = HumanReadableBytes.parse("100MiB");
@@ -305,11 +306,12 @@ class QueryVirtualStorageTest extends
EmbeddedClusterTestBase
@Test
void testQuerySysTables()
{
- String query = "SELECT curr_size, max_size, storage_size FROM sys.servers
WHERE tier IS NOT NULL AND server_type = 'historical'";
- Assertions.assertEquals(
- StringUtils.format("%s,%s,%s", SIZE_BYTES, MAX_SIZE, CACHE_SIZE),
- cluster.callApi().runSql(query)
- );
+ final String query = "SELECT curr_size, max_size, storage_size FROM
sys.servers WHERE tier IS NOT NULL AND server_type = 'historical'";
+ final String resultString = cluster.callApi().runSql(query);
+ final String[] split = resultString.split(",");
+ Assertions.assertTrue(Long.parseLong(split[0]) <= SIZE_BYTES);
+ Assertions.assertEquals(MAX_SIZE, Long.parseLong(split[1]));
+ Assertions.assertEquals(CACHE_SIZE, Long.parseLong(split[2]));
}
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
index 470ba256deb..0f219fe76f7 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexSpec.java
@@ -29,6 +29,7 @@ import org.apache.druid.segment.data.BitmapSerde;
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.CompressionFactory;
import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.file.SegmentFileBuilderV10;
import org.apache.druid.segment.loading.SegmentizerFactory;
import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec;
@@ -249,7 +250,7 @@ public class IndexSpec
} else if (defaultSpec.metadataCompression != null) {
bob.withMetadataCompression(defaultSpec.metadataCompression);
} else {
- bob.withMetadataCompression(CompressionStrategy.NONE);
+
bob.withMetadataCompression(SegmentFileBuilderV10.DEFAULT_METADATA_COMPRESSION);
}
if (dimensionCompression != null) {
diff --git
a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileBuilderV10.java
b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileBuilderV10.java
index 0557fc2a4bd..0b17960aa57 100644
---
a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileBuilderV10.java
+++
b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileBuilderV10.java
@@ -80,9 +80,14 @@ public class SegmentFileBuilderV10 implements
SegmentFileBuilder
{
private static final Logger LOG = new Logger(SegmentFileBuilderV10.class);
+ /**
+ * Default compression for the V10 metadata header
+ */
+ public static final CompressionStrategy DEFAULT_METADATA_COMPRESSION =
CompressionStrategy.ZSTD;
+
public static SegmentFileBuilderV10 create(ObjectMapper jsonMapper, File
baseDir)
{
- return create(jsonMapper, baseDir, CompressionStrategy.NONE);
+ return create(jsonMapper, baseDir, DEFAULT_METADATA_COMPRESSION);
}
public static SegmentFileBuilderV10 create(ObjectMapper jsonMapper, File
baseDir, CompressionStrategy metaCompression)
@@ -184,6 +189,7 @@ public class SegmentFileBuilderV10 implements
SegmentFileBuilder
if (internalFiles.containsKey(name)) {
throw new IAE("Cannot add files of the same name, already have [%s]",
name);
}
+ ensureNameMatchesActiveGroup(name);
if (size > maxContainerSize) {
throw DruidException.forPersona(DruidException.Persona.ADMIN)
.ofCategory(DruidException.Category.RUNTIME_FAILURE)
@@ -285,9 +291,29 @@ public class SegmentFileBuilderV10 implements
SegmentFileBuilder
@Override
public void addColumn(String name, ColumnDescriptor columnDescriptor)
{
+ ensureNameMatchesActiveGroup(name);
this.columns.put(name, columnDescriptor);
}
+ /**
+ * If a file group is currently active (set by the most recent {@link
#startFileGroup} call), enforce that names of
+ * files and columns added under it are prefixed by {@code groupName + "/"}.
Prevents silent collisions where two
+ * groups write a file/column of the same bare name and the second silently
overwrites the first in the metadata
+ * maps. Existing production callers (e.g. {@code IndexMergerV10} via
+ * {@code Projections.getProjectionSegmentInternalFileName}) already
construct prefixed names, so this is a no-op
+ * for them; it catches new writers that forget the convention.
+ */
+ private void ensureNameMatchesActiveGroup(String name)
+ {
+ if (currentFileGroup != null && !name.startsWith(currentFileGroup + "/")) {
+ throw DruidException.defensive(
+ "Name[%s] must start with the active file group prefix[%s/]",
+ name,
+ currentFileGroup
+ );
+ }
+ }
+
/**
* Declare the file group that subsequent writes belong to. Writes are
routed into a container tagged with the
* declared group; a new container is rolled when the group changes or the
incoming file won't fit. A group whose
@@ -438,7 +464,7 @@ public class SegmentFileBuilderV10 implements
SegmentFileBuilder
long offset = 0;
for (ContainerWriter container : containers) {
final long length = container.file.length();
- result.add(new SegmentFileContainerMetadata(offset, length));
+ result.add(new SegmentFileContainerMetadata(offset, length,
container.group));
offset += length;
}
return result;
diff --git
a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileContainerMetadata.java
b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileContainerMetadata.java
index bd7a414a385..3739eb7718b 100644
---
a/processing/src/main/java/org/apache/druid/segment/file/SegmentFileContainerMetadata.java
+++
b/processing/src/main/java/org/apache/druid/segment/file/SegmentFileContainerMetadata.java
@@ -20,25 +20,40 @@
package org.apache.druid.segment.file;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import javax.annotation.Nullable;
+import java.util.Objects;
+
/**
* Starting offset and size of a 'container' stored in a V10 segment file;
think the V10 equivalent of V9's external
* 'smoosh' files, e.g. 00000.smoosh.
+ * <p>
+ * Each container holds internal files belonging to at most one named file
group, as declared at write time via
+ * {@link SegmentFileBuilder#startFileGroup}. The {@link #fileGroup} field
records that name so readers can attribute
+ * a container to its group without parsing internal-file names. The field is
{@code null} for containers written
+ * without a {@code startFileGroup} call (or with {@code
startFileGroup(null)}), and for containers from segments
+ * produced by writers that pre-date this field; null serializes as a
Jackson-omitted property so old segments
+ * round-trip unchanged.
*/
public class SegmentFileContainerMetadata
{
private final long startOffset;
private final long size;
+ @Nullable
+ private final String fileGroup;
@JsonCreator
public SegmentFileContainerMetadata(
@JsonProperty("startOffset") long startOffset,
- @JsonProperty("size") long size
+ @JsonProperty("size") long size,
+ @JsonProperty("fileGroup") @Nullable String fileGroup
)
{
this.startOffset = startOffset;
this.size = size;
+ this.fileGroup = fileGroup;
}
@JsonProperty
@@ -52,4 +67,43 @@ public class SegmentFileContainerMetadata
{
return size;
}
+
+ @JsonProperty
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @Nullable
+ public String getFileGroup()
+ {
+ return fileGroup;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SegmentFileContainerMetadata that = (SegmentFileContainerMetadata) o;
+ return startOffset == that.startOffset
+ && size == that.size
+ && Objects.equals(fileGroup, that.fileGroup);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(startOffset, size, fileGroup);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SegmentFileContainerMetadata{"
+ + "startOffset=" + startOffset
+ + ", size=" + size
+ + ", fileGroup=" + fileGroup
+ + '}';
+ }
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/file/SegmentFileBuilderV10Test.java
b/processing/src/test/java/org/apache/druid/segment/file/SegmentFileBuilderV10Test.java
index 89c232973cc..6dd01d8e5bd 100644
---
a/processing/src/test/java/org/apache/druid/segment/file/SegmentFileBuilderV10Test.java
+++
b/processing/src/test/java/org/apache/druid/segment/file/SegmentFileBuilderV10Test.java
@@ -26,6 +26,8 @@ import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.IndexIO;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnDescriptor;
+import org.apache.druid.segment.column.ValueType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -49,8 +51,7 @@ class SegmentFileBuilderV10Test
@Test
void testOneContainerPerProjection() throws IOException
{
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
// matches the production usage pattern in IndexMergerV10: call
startFileGroup then write that projection's
// columns, then move on to the next projection.
@@ -83,8 +84,7 @@ class SegmentFileBuilderV10Test
@Test
void testProjectionNameWithSlashRoutesCorrectly() throws IOException
{
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
final String slashyProjection = "nested/projection";
final int colCount = 3;
@@ -127,11 +127,157 @@ class SegmentFileBuilderV10Test
}
}
+ @Test
+ void testAddWithoutGroupPrefixThrowsWhenGroupActive() throws IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ builder.startFileGroup("projA");
+ final File tmp = new File(tempDir, "no-prefix.bin");
+ Files.write(Ints.toByteArray(1), tmp);
+ // file name doesn't start with "projA/", so add must throw
+ Assertions.assertThrows(RuntimeException.class, () ->
builder.add("wrong/col0", tmp));
+ }
+ }
+
+ @Test
+ void testAddWithChannelWithoutGroupPrefixThrowsWhenGroupActive() throws
IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ builder.startFileGroup("projA");
+ Assertions.assertThrows(RuntimeException.class, () ->
builder.addWithChannel("wrong/col0", 4));
+ }
+ }
+
+ @Test
+ void testAddColumnWithoutGroupPrefixThrowsWhenGroupActive() throws
IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ builder.startFileGroup("projA");
+ Assertions.assertThrows(
+ RuntimeException.class,
+ () -> builder.addColumn("wrong_no_prefix", new
ColumnDescriptor.Builder()
+ .setValueType(ValueType.LONG)
+ .build())
+ );
+ }
+ }
+
+ @Test
+ void testAddWithoutPrefixIsAllowedWhenNoGroupActive() throws IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ // never call startFileGroup; bare names are fine
+ final File tmp = new File(tempDir, "bare.bin");
+ Files.write(Ints.toByteArray(1), tmp);
+ builder.add("col0", tmp);
+ }
+ // success: no exception
+ }
+
+ @Test
+ void testContainerMetadataCarriesFileGroup() throws IOException
+ {
+ final File baseDir = newBaseDir();
+
+ final String[] projections = {"__base", "projA", "projB"};
+ final int colCount = 2;
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ for (String projection : projections) {
+ builder.startFileGroup(projection);
+ for (int col = 0; col < colCount; col++) {
+ final String name = projection + "/col" + col;
+ final File tmpFile = new File(tempDir,
StringUtils.format("%s-%s.bin", projection, col));
+ Files.write(Ints.toByteArray(name.hashCode()), tmpFile);
+ builder.add(name, tmpFile);
+ }
+ }
+ }
+
+ final File segmentFile = new File(baseDir, IndexIO.V10_FILE_NAME);
+ try (SegmentFileMapperV10 mapper =
SegmentFileMapperV10.create(segmentFile, JSON_MAPPER)) {
+ final SegmentFileMetadata metadata = mapper.getSegmentFileMetadata();
+ Assertions.assertEquals(projections.length,
metadata.getContainers().size());
+
+ // Each container's fileGroup must equal the group active when it was
written. Build the expected list by
+ // walking the files: each container holds files from exactly one group,
so the first file's group prefix is
+ // authoritative.
+ for (int ci = 0; ci < metadata.getContainers().size(); ci++) {
+ final int containerIdx = ci;
+ final String expectedGroup = metadata.getFiles().entrySet().stream()
+ .filter(e -> e.getValue().getContainer() == containerIdx)
+ .map(e -> e.getKey().substring(0, e.getKey().indexOf('/')))
+ .findFirst()
+ .orElseThrow();
+ Assertions.assertEquals(
+ expectedGroup,
+ metadata.getContainers().get(ci).getFileGroup(),
+ "container " + ci + " fileGroup mismatch"
+ );
+ }
+ }
+ }
+
+ @Test
+ void testContainerWrittenWithoutStartFileGroupHasNullFileGroup() throws
IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ // never call startFileGroup; the single container should carry
fileGroup == null
+ for (int col = 0; col < 3; col++) {
+ final String name = "col" + col;
+ final File tmpFile = new File(tempDir,
StringUtils.format("nogroup-%s.bin", col));
+ Files.write(Ints.toByteArray(name.hashCode()), tmpFile);
+ builder.add(name, tmpFile);
+ }
+ }
+
+ final File segmentFile = new File(baseDir, IndexIO.V10_FILE_NAME);
+ try (SegmentFileMapperV10 mapper =
SegmentFileMapperV10.create(segmentFile, JSON_MAPPER)) {
+ final SegmentFileMetadata metadata = mapper.getSegmentFileMetadata();
+ Assertions.assertEquals(1, metadata.getContainers().size());
+ Assertions.assertNull(metadata.getContainers().get(0).getFileGroup());
+ }
+ }
+
+ @Test
+ void testStartFileGroupNullClearsCurrentGroup() throws IOException
+ {
+ final File baseDir = newBaseDir();
+
+ try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
+ builder.startFileGroup("first");
+ final File firstFile = new File(tempDir, "first.bin");
+ Files.write(Ints.toByteArray(1), firstFile);
+ builder.add("first/a", firstFile);
+
+ builder.startFileGroup(null);
+ final File noGroupFile = new File(tempDir, "ng.bin");
+ Files.write(Ints.toByteArray(2), noGroupFile);
+ builder.add("ng/a", noGroupFile);
+ }
+
+ final File segmentFile = new File(baseDir, IndexIO.V10_FILE_NAME);
+ try (SegmentFileMapperV10 mapper =
SegmentFileMapperV10.create(segmentFile, JSON_MAPPER)) {
+ final SegmentFileMetadata metadata = mapper.getSegmentFileMetadata();
+ Assertions.assertEquals(2, metadata.getContainers().size());
+ Assertions.assertEquals("first",
metadata.getContainers().get(0).getFileGroup());
+ Assertions.assertNull(metadata.getContainers().get(1).getFileGroup());
+ }
+ }
+
@Test
void testStartFileGroupWhileWriterInUseThrows() throws IOException
{
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
builder.startFileGroup("__base");
@@ -146,8 +292,7 @@ class SegmentFileBuilderV10Test
void testExternalBuilderAlsoSplitsContainersByProjection() throws IOException
{
final String externalName = "external.segment";
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
final String[] mainProjections = {"__base", "projA", "projB"};
final String[] externalProjections = {"extProjX", "extProjY"};
@@ -219,8 +364,7 @@ class SegmentFileBuilderV10Test
// merged back in at outer-close. Main and external each drive this
independently, and since they share baseDir,
// their delegate file names must not collide.
final String externalName = "external.segment";
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
final byte[] outerBytes = new byte[]{1, 2, 3, 4};
final byte[] nestedBytes = new byte[]{5, 6, 7, 8};
@@ -261,8 +405,7 @@ class SegmentFileBuilderV10Test
// while file group "groupA" was active; even if the caller switches to
"groupB" before finally closing the nested
// channel, the delegated bytes must still land in groupA's container, not
groupB's. Otherwise the grouping breaks,
// and files from other groups end up in the same container.
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
final byte[] outerBytes = new byte[]{1, 2, 3, 4};
final byte[] nestedBytes = new byte[]{5, 6, 7, 8};
@@ -310,8 +453,7 @@ class SegmentFileBuilderV10Test
@Test
void testUnprefixedFilesShareSingleContainer() throws IOException
{
- final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
- FileUtils.mkdirp(baseDir);
+ final File baseDir = newBaseDir();
try (SegmentFileBuilderV10 builder =
SegmentFileBuilderV10.create(JSON_MAPPER, baseDir)) {
for (int i = 0; i < 5; ++i) {
@@ -336,6 +478,13 @@ class SegmentFileBuilderV10Test
Assertions.assertArrayEquals(expected, got);
}
+ private File newBaseDir() throws IOException
+ {
+ final File baseDir = new File(tempDir, "base_" +
ThreadLocalRandom.current().nextInt());
+ FileUtils.mkdirp(baseDir);
+ return baseDir;
+ }
+
private static void assertNoContainerMixesProjections(SegmentFileMetadata
metadata)
{
for (int containerIdx = 0; containerIdx < metadata.getContainers().size();
containerIdx++) {
diff --git
a/processing/src/test/java/org/apache/druid/segment/file/SegmentFileContainerMetadataTest.java
b/processing/src/test/java/org/apache/druid/segment/file/SegmentFileContainerMetadataTest.java
new file mode 100644
index 00000000000..5a56dcd7faf
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/file/SegmentFileContainerMetadataTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.file;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.segment.TestHelper;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class SegmentFileContainerMetadataTest
+{
+ private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
+
+ @Test
+ void testEqualsAndHashCode()
+ {
+
EqualsVerifier.forClass(SegmentFileContainerMetadata.class).usingGetClass().verify();
+ }
+
+ @Test
+ void testSerdeWithFileGroup() throws Exception
+ {
+ final SegmentFileContainerMetadata metadata = new
SegmentFileContainerMetadata(100, 4096, "projA");
+ final String json = JSON_MAPPER.writeValueAsString(metadata);
+ Assertions.assertTrue(json.contains("\"fileGroup\":\"projA\""), "fileGroup
must be present in serialized JSON: " + json);
+ Assertions.assertEquals(metadata, JSON_MAPPER.readValue(json,
SegmentFileContainerMetadata.class));
+ }
+
+ @Test
+ void testSerdeWithNullFileGroupOmitsField() throws Exception
+ {
+ // Old-format segments don't have fileGroup; serializing null must omit
the property so older readers (and
+ // future versions reading old segments) round-trip unchanged.
+ final SegmentFileContainerMetadata metadata = new
SegmentFileContainerMetadata(0, 1024, null);
+ final String json = JSON_MAPPER.writeValueAsString(metadata);
+ Assertions.assertFalse(json.contains("fileGroup"), "null fileGroup must be
omitted from JSON, got: " + json);
+ Assertions.assertEquals(metadata, JSON_MAPPER.readValue(json,
SegmentFileContainerMetadata.class));
+ }
+
+ @Test
+ void testDeserializeLegacyJsonWithoutFileGroup() throws Exception
+ {
+ // Bytes produced by a writer pre-dating the fileGroup field must
deserialize cleanly with fileGroup == null.
+ final String legacyJson = "{\"startOffset\":42,\"size\":8192}";
+ final SegmentFileContainerMetadata metadata =
JSON_MAPPER.readValue(legacyJson, SegmentFileContainerMetadata.class);
+ Assertions.assertEquals(42, metadata.getStartOffset());
+ Assertions.assertEquals(8192, metadata.getSize());
+ Assertions.assertNull(metadata.getFileGroup());
+ }
+}
diff --git
a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
index 04cee38e87c..6dfd3ff1a1f 100644
---
a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
+++
b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java
@@ -298,12 +298,12 @@ public class CompactionStatusTest
compactionConfig,
"'indexSpec' mismatch: "
+ "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{},"
- + " metadataCompression=none,"
+ + " metadataCompression=zstd,"
+ " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{},"
+ " metricCompression=lz4, longEncoding=longs,
complexMetricCompression=null,"
+ " autoColumnFormatSpec=null, stringColumnFormatSpec=null,
jsonCompression=null, segmentLoader=null}], "
+ "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{},"
- + " metadataCompression=none,"
+ + " metadataCompression=zstd,"
+ " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{},"
+ " metricCompression=lz4, longEncoding=longs,
complexMetricCompression=null,"
+ " autoColumnFormatSpec=null, stringColumnFormatSpec=null,
jsonCompression=null, segmentLoader=null}]"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
