This is an automated email from the ASF dual-hosted git repository. JingsongLi pushed a commit to branch codex/pk-vector-01-foundation in repository https://gitbox.apache.org/repos/asf/paimon.git
commit cd7b714ab804f46c54d5b058d60adb78f3e439a3 Author: JingsongLi <[email protected]> AuthorDate: Sat Jul 11 18:37:49 2026 +0800 [core] Minimize primary-key vector segment metadata --- .../index/pkvector/PkVectorAnnSegmentFile.java | 26 +--- .../index/pkvector/PkVectorAnnSegmentSearcher.java | 21 +-- .../index/pkvector/PkVectorRawSegmentFile.java | 36 +---- .../paimon/index/pkvector/PkVectorSegmentMeta.java | 164 +-------------------- .../index/pkvector/PkVectorAnnSegmentFileTest.java | 45 +++--- .../index/pkvector/PkVectorSegmentMetaTest.java | 43 +----- 6 files changed, 43 insertions(+), 292 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java index e899fc07d8..110c221440 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java @@ -47,7 +47,6 @@ import java.util.Locale; import java.util.Map; import java.util.function.LongPredicate; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Builds immutable ANN payloads whose index ids are source data-file row positions. */ @@ -65,11 +64,8 @@ public class PkVectorAnnSegmentFile extends IndexFile { DataField vectorField, Options indexOptions, String indexDefinitionId, - String vectorTypeFingerprint, String metric, - String algorithm, - byte[] optionsHash, - long buildSnapshotId) + String algorithm) throws IOException { checkArgument(!sources.isEmpty(), "An ANN segment must reference source files."); long totalRowCount = 0; @@ -157,18 +153,7 @@ public class PkVectorAnnSegmentFile extends IndexFile { byte[] payloadMetadata = result.meta() == null ? new byte[0] : result.meta(); PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, - indexDefinitionId, - vectorField.id(), - vectorTypeFingerprint, - normalizeMetric(metric), - algorithm, - sourceFiles, - ordinalLayout, - liveRowCount, - buildSnapshotId, - optionsHash, - payloadMetadata); + indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); IndexFileMeta segment = new IndexFileMeta( PK_VECTOR_ANN, @@ -191,12 +176,7 @@ public class PkVectorAnnSegmentFile extends IndexFile { } private static PkVectorSegmentMeta.SourceFile sourceMetadata(DataFileMeta sourceFile) { - return new PkVectorSegmentMeta.SourceFile( - sourceFile.fileName(), - sourceFile.schemaId(), - sourceFile.level(), - sourceFile.rowCount(), - sourceFile.fileSize()); + return new PkVectorSegmentMeta.SourceFile(sourceFile.fileName(), sourceFile.rowCount()); } private static String normalizeMetric(String metric) { diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java index 2ba7daae35..5ed2b2e548 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java @@ -47,7 +47,6 @@ import java.util.concurrent.ExecutorService; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Searches one ANN payload and maps its segment-local ids back to source row positions. */ @@ -62,6 +61,8 @@ public class PkVectorAnnSegmentSearcher { private final PkVectorAnnSegmentFile annSegmentFile; private final DataField vectorField; private final Options indexOptions; + private final String algorithm; + private final String metric; private final ExecutorService executor; public PkVectorAnnSegmentSearcher( @@ -69,11 +70,15 @@ public class PkVectorAnnSegmentSearcher { PkVectorAnnSegmentFile annSegmentFile, DataField vectorField, Options indexOptions, + String algorithm, + String metric, ExecutorService executor) { this.fileIO = fileIO; this.annSegmentFile = annSegmentFile; this.vectorField = vectorField; this.indexOptions = indexOptions; + this.algorithm = algorithm; + this.metric = normalizeMetric(metric); this.executor = executor; } @@ -106,7 +111,6 @@ public class PkVectorAnnSegmentSearcher { PkVectorAnnSegmentFile.PK_VECTOR_ANN.equals(segment.indexType()), "Vector segment %s is not an ANN payload.", segment.fileName()); - checkArgument(metadata.role() == ANN, "Vector segment %s is not ANN.", segment.fileName()); checkArgument( metadata.ordinalLayout() == ROW_POSITION || metadata.ordinalLayout() == FILE_POSITION, @@ -117,20 +121,11 @@ public class PkVectorAnnSegmentSearcher { metadata.ordinalLayout() != ROW_POSITION || metadata.sourceFiles().size() == 1, "Row-position ANN segment %s must reference exactly one source file.", segment.fileName()); - checkArgument( - metadata.vectorFieldId() == vectorField.id(), - "ANN segment %s has vector field %s, but reader expects %s.", - segment.fileName(), - metadata.vectorFieldId(), - vectorField.id()); - - GlobalIndexer indexer = - GlobalIndexer.create(metadata.algorithm(), vectorField, indexOptions); + GlobalIndexer indexer = GlobalIndexer.create(algorithm, vectorField, indexOptions); checkArgument( indexer instanceof VectorGlobalIndexer, "Index algorithm %s does not implement VectorGlobalIndexer.", - metadata.algorithm()); - String metric = normalizeMetric(metadata.metric()); + algorithm); String readerMetric = normalizeMetric(((VectorGlobalIndexer) indexer).metric()); checkArgument( metric.equals(readerMetric), diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java index a8d0b129d2..e1dd7b1c90 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorRawSegmentFile.java @@ -40,7 +40,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.RAW_DELTA; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -61,10 +60,6 @@ public class PkVectorRawSegmentFile extends IndexFile { int dimension, String indexDefinitionId, int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, BiConsumer<IndexFileMeta, FileSource> segmentConsumer, Consumer<IndexFileMeta> segmentAbortConsumer) { Path path = pathFactory.newPath(); @@ -73,10 +68,6 @@ public class PkVectorRawSegmentFile extends IndexFile { new RawVectorSidecarWriter(fileIO, path, dimension), indexDefinitionId, vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - optionsHash, segmentConsumer, segmentAbortConsumer); } catch (IOException e) { @@ -91,10 +82,6 @@ public class PkVectorRawSegmentFile extends IndexFile { private final RawVectorSidecarWriter rawWriter; private final String indexDefinitionId; private final int vectorFieldId; - private final String vectorTypeFingerprint; - private final String metric; - private final String algorithm; - private final byte[] optionsHash; private final BiConsumer<IndexFileMeta, FileSource> segmentConsumer; private final Consumer<IndexFileMeta> segmentAbortConsumer; @@ -106,19 +93,11 @@ public class PkVectorRawSegmentFile extends IndexFile { RawVectorSidecarWriter rawWriter, String indexDefinitionId, int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - byte[] optionsHash, BiConsumer<IndexFileMeta, FileSource> segmentConsumer, Consumer<IndexFileMeta> segmentAbortConsumer) { this.rawWriter = rawWriter; this.indexDefinitionId = indexDefinitionId; this.vectorFieldId = vectorFieldId; - this.vectorTypeFingerprint = vectorTypeFingerprint; - this.metric = metric; - this.algorithm = algorithm; - this.optionsHash = optionsHash.clone(); this.segmentConsumer = segmentConsumer; this.segmentAbortConsumer = segmentAbortConsumer; } @@ -149,23 +128,12 @@ public class PkVectorRawSegmentFile extends IndexFile { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - RAW_DELTA, indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, Collections.singletonList( new PkVectorSegmentMeta.SourceFile( - sourceFile.fileName(), - sourceFile.schemaId(), - sourceFile.level(), - sourceFile.rowCount(), - sourceFile.fileSize())), + sourceFile.fileName(), sourceFile.rowCount())), ROW_POSITION, - rawWriter.liveVectorCount(), - 0, - optionsHash); + new byte[0]); Path path = rawWriter.path(); IndexFileMeta segment = new IndexFileMeta( diff --git a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java index c9831f7415..8dc00d49a1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorSegmentMeta.java @@ -35,101 +35,28 @@ public class PkVectorSegmentMeta { private static final int VERSION = 1; - private final Role role; private final String indexDefinitionId; - private final int vectorFieldId; - private final String vectorTypeFingerprint; - private final String metric; - private final String algorithm; private final List<SourceFile> sourceFiles; private final OrdinalLayout ordinalLayout; - private final long liveRowCountAtBuild; - private final long buildSnapshotId; - private final byte[] optionsHash; private final byte[] payloadMetadata; public PkVectorSegmentMeta( - Role role, String indexDefinitionId, - int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, List<SourceFile> sourceFiles, OrdinalLayout ordinalLayout, - long liveRowCountAtBuild, - long buildSnapshotId, - byte[] optionsHash) { - this( - role, - indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - sourceFiles, - ordinalLayout, - liveRowCountAtBuild, - buildSnapshotId, - optionsHash, - new byte[0]); - } - - public PkVectorSegmentMeta( - Role role, - String indexDefinitionId, - int vectorFieldId, - String vectorTypeFingerprint, - String metric, - String algorithm, - List<SourceFile> sourceFiles, - OrdinalLayout ordinalLayout, - long liveRowCountAtBuild, - long buildSnapshotId, - byte[] optionsHash, byte[] payloadMetadata) { - this.role = Objects.requireNonNull(role); this.indexDefinitionId = Objects.requireNonNull(indexDefinitionId); - this.vectorFieldId = vectorFieldId; - this.vectorTypeFingerprint = Objects.requireNonNull(vectorTypeFingerprint); - this.metric = Objects.requireNonNull(metric); - this.algorithm = Objects.requireNonNull(algorithm); this.sourceFiles = Collections.unmodifiableList(new ArrayList<>(sourceFiles)); this.ordinalLayout = Objects.requireNonNull(ordinalLayout); - this.liveRowCountAtBuild = liveRowCountAtBuild; - this.buildSnapshotId = buildSnapshotId; - this.optionsHash = Arrays.copyOf(optionsHash, optionsHash.length); this.payloadMetadata = Arrays.copyOf(payloadMetadata, payloadMetadata.length); checkArgument(!this.sourceFiles.isEmpty(), "A vector segment must reference source files."); - checkArgument(liveRowCountAtBuild >= 0, "Live row count must not be negative."); - checkArgument(buildSnapshotId >= 0, "Build snapshot id must not be negative."); - } - - public Role role() { - return role; } public String indexDefinitionId() { return indexDefinitionId; } - public int vectorFieldId() { - return vectorFieldId; - } - - public String vectorTypeFingerprint() { - return vectorTypeFingerprint; - } - - public String metric() { - return metric; - } - - public String algorithm() { - return algorithm; - } - public List<SourceFile> sourceFiles() { return sourceFiles; } @@ -138,18 +65,6 @@ public class PkVectorSegmentMeta { return ordinalLayout; } - public long liveRowCountAtBuild() { - return liveRowCountAtBuild; - } - - public long buildSnapshotId() { - return buildSnapshotId; - } - - public byte[] optionsHash() { - return Arrays.copyOf(optionsHash, optionsHash.length); - } - public byte[] payloadMetadata() { return Arrays.copyOf(payloadMetadata, payloadMetadata.length); } @@ -159,25 +74,13 @@ public class PkVectorSegmentMeta { try { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(VERSION); - output.writeByte(role.ordinal()); output.writeUTF(indexDefinitionId); - output.writeInt(vectorFieldId); - output.writeUTF(vectorTypeFingerprint); - output.writeUTF(metric); - output.writeUTF(algorithm); output.writeInt(sourceFiles.size()); for (SourceFile sourceFile : sourceFiles) { output.writeUTF(sourceFile.fileName); - output.writeLong(sourceFile.schemaId); - output.writeInt(sourceFile.level); output.writeLong(sourceFile.rowCount); - output.writeLong(sourceFile.fileSize); } output.writeByte(ordinalLayout.ordinal()); - output.writeLong(liveRowCountAtBuild); - output.writeLong(buildSnapshotId); - output.writeInt(optionsHash.length); - output.write(optionsHash); output.writeInt(payloadMetadata.length); output.write(payloadMetadata); return output.getCopyOfBuffer(); @@ -196,32 +99,15 @@ public class PkVectorSegmentMeta { version == VERSION, "Unsupported primary-key vector segment version: %s.", version); - Role role = enumValue(Role.values(), input.readByte(), "role"); String indexDefinitionId = input.readUTF(); - int vectorFieldId = input.readInt(); - String vectorTypeFingerprint = input.readUTF(); - String metric = input.readUTF(); - String algorithm = input.readUTF(); int sourceFileCount = input.readInt(); checkArgument(sourceFileCount > 0, "A vector segment must reference source files."); List<SourceFile> sourceFiles = new ArrayList<>(sourceFileCount); for (int i = 0; i < sourceFileCount; i++) { - sourceFiles.add( - new SourceFile( - input.readUTF(), - input.readLong(), - input.readInt(), - input.readLong(), - input.readLong())); + sourceFiles.add(new SourceFile(input.readUTF(), input.readLong())); } OrdinalLayout ordinalLayout = enumValue(OrdinalLayout.values(), input.readByte(), "ordinal layout"); - long liveRowCountAtBuild = input.readLong(); - long buildSnapshotId = input.readLong(); - int optionsHashLength = input.readInt(); - checkArgument(optionsHashLength >= 0, "Options hash length must not be negative."); - byte[] optionsHash = new byte[optionsHashLength]; - input.readFully(optionsHash); int payloadMetadataLength = input.readInt(); checkArgument( payloadMetadataLength >= 0, "Payload metadata length must not be negative."); @@ -231,18 +117,7 @@ public class PkVectorSegmentMeta { input.available() == 0, "Unexpected trailing bytes in vector segment metadata."); return new PkVectorSegmentMeta( - role, - indexDefinitionId, - vectorFieldId, - vectorTypeFingerprint, - metric, - algorithm, - sourceFiles, - ordinalLayout, - liveRowCountAtBuild, - buildSnapshotId, - optionsHash, - payloadMetadata); + indexDefinitionId, sourceFiles, ordinalLayout, payloadMetadata); } catch (IOException e) { throw new IllegalArgumentException( "Failed to deserialize primary-key vector segment metadata.", e); @@ -259,12 +134,6 @@ public class PkVectorSegmentMeta { return values[index]; } - /** Role of an immutable vector payload. */ - public enum Role { - RAW_DELTA, - ANN - } - /** Mapping from a segment-local ordinal to a physical data-file position. */ public enum OrdinalLayout { ROW_POSITION, @@ -275,41 +144,22 @@ public class PkVectorSegmentMeta { public static class SourceFile { private final String fileName; - private final long schemaId; - private final int level; private final long rowCount; - private final long fileSize; - public SourceFile(String fileName, long schemaId, int level, long rowCount, long fileSize) { + public SourceFile(String fileName, long rowCount) { this.fileName = Objects.requireNonNull(fileName); - this.schemaId = schemaId; - this.level = level; this.rowCount = rowCount; - this.fileSize = fileSize; checkArgument(rowCount >= 0, "Source file row count must not be negative."); - checkArgument(fileSize >= 0, "Source file size must not be negative."); } public String fileName() { return fileName; } - public long schemaId() { - return schemaId; - } - - public int level() { - return level; - } - public long rowCount() { return rowCount; } - public long fileSize() { - return fileSize; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -319,16 +169,12 @@ public class PkVectorSegmentMeta { return false; } SourceFile that = (SourceFile) o; - return schemaId == that.schemaId - && level == that.level - && rowCount == that.rowCount - && fileSize == that.fileSize - && Objects.equals(fileName, that.fileName); + return rowCount == that.rowCount && Objects.equals(fileName, that.fileName); } @Override public int hashCode() { - return Objects.hash(fileName, schemaId, level, rowCount, fileSize); + return Objects.hash(fileName, rowCount); } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java index 64c1e93b0f..dbb49d1e2f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java @@ -44,7 +44,6 @@ import java.util.concurrent.Executors; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.ROW_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.assertj.core.api.Assertions.assertThat; /** Tests ANN payload construction through the vector GlobalIndexer SPI. */ @@ -78,11 +77,8 @@ class PkVectorAnnSegmentFileTest { vectorField, options, "definition", - "ARRAY<FLOAT>", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } assertThat(segment.indexType()).isEqualTo(PkVectorAnnSegmentFile.PK_VECTOR_ANN); @@ -90,13 +86,10 @@ class PkVectorAnnSegmentFileTest { assertThat(fileIO.exists(pathFactory.toPath(segment))).isTrue(); PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.role()).isEqualTo(ANN); assertThat(metadata.ordinalLayout()).isEqualTo(ROW_POSITION); assertThat(metadata.sourceFiles()).hasSize(1); assertThat(metadata.sourceFiles().get(0).fileName()).isEqualTo("data-1"); - assertThat(metadata.liveRowCountAtBuild()).isEqualTo(2); - assertThat(metadata.buildSnapshotId()).isEqualTo(42); - assertThat(metadata.optionsHash()).containsExactly(1, 2); + assertThat(segment.globalIndexMeta().indexFieldId()).isEqualTo(7); } @Test @@ -128,16 +121,12 @@ class PkVectorAnnSegmentFileTest { vectorField, options, "definition", - "ARRAY<FLOAT>", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); - assertThat(metadata.liveRowCountAtBuild()).isEqualTo(1); assertThat(segment.rowCount()).isEqualTo(1); } @@ -167,11 +156,8 @@ class PkVectorAnnSegmentFileTest { vectorField, options, "definition", - "ARRAY<FLOAT>", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = PkVectorSegmentMeta.deserialize(segment.globalIndexMeta().indexMeta()); @@ -181,7 +167,14 @@ class PkVectorAnnSegmentFileTest { List<PkVectorAnnSegmentSearcher.Candidate> candidates; try { candidates = - new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + new PkVectorAnnSegmentSearcher( + fileIO, + annFile, + vectorField, + options, + "test-vector-ann", + "l2", + executor) .search( segment, metadata, @@ -231,11 +224,8 @@ class PkVectorAnnSegmentFileTest { vectorField, options, "definition", - "ARRAY<FLOAT>", "l2", - "test-vector-ann", - new byte[] {1, 2}, - 42); + "test-vector-ann"); } PkVectorSegmentMeta metadata = @@ -255,7 +245,14 @@ class PkVectorAnnSegmentFileTest { List<PkVectorAnnSegmentSearcher.Candidate> candidates; try { candidates = - new PkVectorAnnSegmentSearcher(fileIO, annFile, vectorField, options, executor) + new PkVectorAnnSegmentSearcher( + fileIO, + annFile, + vectorField, + options, + "test-vector-ann", + "l2", + executor) .search( segment, metadata, diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java index afb0f068ed..ece36b55eb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSegmentMetaTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.OrdinalLayout.FILE_POSITION; -import static org.apache.paimon.index.pkvector.PkVectorSegmentMeta.Role.ANN; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -36,34 +35,18 @@ class PkVectorSegmentMetaTest { void testRoundTrip() { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, "1d4502f1-9cf0-4d86-8d8d-5cc9ac05e108", - 7, - "VECTOR<FLOAT>(1024)", - "l2", - "ivf-pq", Arrays.asList( - new PkVectorSegmentMeta.SourceFile("data-1", 3, 0, 100, 1024), - new PkVectorSegmentMeta.SourceFile("data-2", 3, 0, 50, 512)), + new PkVectorSegmentMeta.SourceFile("data-1", 100), + new PkVectorSegmentMeta.SourceFile("data-2", 50)), FILE_POSITION, - 120, - 42, - new byte[] {1, 2, 3}, new byte[] {4, 5, 6}); PkVectorSegmentMeta restored = PkVectorSegmentMeta.deserialize(metadata.serialize()); - assertThat(restored.role()).isEqualTo(ANN); assertThat(restored.indexDefinitionId()).isEqualTo(metadata.indexDefinitionId()); - assertThat(restored.vectorFieldId()).isEqualTo(7); - assertThat(restored.vectorTypeFingerprint()).isEqualTo("VECTOR<FLOAT>(1024)"); - assertThat(restored.metric()).isEqualTo("l2"); - assertThat(restored.algorithm()).isEqualTo("ivf-pq"); assertThat(restored.sourceFiles()).isEqualTo(metadata.sourceFiles()); assertThat(restored.ordinalLayout()).isEqualTo(FILE_POSITION); - assertThat(restored.liveRowCountAtBuild()).isEqualTo(120); - assertThat(restored.buildSnapshotId()).isEqualTo(42); - assertThat(restored.optionsHash()).containsExactly(1, 2, 3); assertThat(restored.payloadMetadata()).containsExactly(4, 5, 6); } @@ -71,16 +54,9 @@ class PkVectorSegmentMetaTest { void testRejectTrailingBytes() { PkVectorSegmentMeta metadata = new PkVectorSegmentMeta( - ANN, "index", - 1, - "VECTOR<FLOAT>(2)", - "l2", - "ivf-pq", - Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1, 0, 1, 8)), + Arrays.asList(new PkVectorSegmentMeta.SourceFile("data", 1)), FILE_POSITION, - 1, - 1, new byte[0]); byte[] bytes = Arrays.copyOf(metadata.serialize(), metadata.serialize().length + 1); @@ -89,26 +65,15 @@ class PkVectorSegmentMetaTest { } @Test - void testRejectsPreReleaseLayoutWithoutPayloadMetadata() throws Exception { + void testRejectsTruncatedPayloadMetadata() throws Exception { DataOutputSerializer output = new DataOutputSerializer(128); output.writeInt(1); - output.writeByte(ANN.ordinal()); output.writeUTF("index"); - output.writeInt(7); - output.writeUTF("VECTOR<FLOAT>(2)"); - output.writeUTF("l2"); - output.writeUTF("ivf-pq"); output.writeInt(1); output.writeUTF("data-1"); - output.writeLong(3); - output.writeInt(0); output.writeLong(10); - output.writeLong(100); output.writeByte(FILE_POSITION.ordinal()); - output.writeLong(9); - output.writeLong(42); output.writeInt(1); - output.writeByte(1); assertThatThrownBy(() -> PkVectorSegmentMeta.deserialize(output.getCopyOfBuffer())) .hasMessageContaining("Failed to deserialize primary-key vector segment metadata");
