This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d3716d1f43 [common] Support VECTOR elements in InternalArray accessors
(#9773)
d3716d1f43 is described below
commit d3716d1f43da3f1509f2fa1f24368d3bbc8cc9b2
Author: YangJie <[email protected]>
AuthorDate: Sun Sep 13 22:39:07 2026 -0400
[common] Support VECTOR elements in InternalArray accessors (#9773)
---
.../java/org/apache/paimon/data/BinaryArray.java | 4 +-
.../java/org/apache/paimon/data/InternalArray.java | 3 +
.../java/org/apache/paimon/data/InternalRow.java | 2 +
.../paimon/data/InternalArrayVectorGetterTest.java | 104 +++++++++++++++++++++
4 files changed, 112 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
index 7a62523711..a2f67e6e17 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryArray.java
@@ -83,6 +83,7 @@ public final class BinaryArray extends BinarySection
implements InternalArray, D
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
case ARRAY:
+ case VECTOR:
case MULTISET:
case MAP:
case ROW:
@@ -260,7 +261,8 @@ public final class BinaryArray extends BinarySection
implements InternalArray, D
@Override
public InternalVector getVector(int pos) {
- throw new IllegalArgumentException("Unsupported type: VectorType");
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readVectorData(segments, offset,
getLong(pos));
}
@Override
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java
b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java
index 78382e8bbd..463f41283b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalArray.java
@@ -124,6 +124,9 @@ public interface InternalArray extends DataGetters {
case ARRAY:
elementGetter = InternalArray::getArray;
break;
+ case VECTOR:
+ elementGetter = InternalArray::getVector;
+ break;
case MULTISET:
case MAP:
elementGetter = InternalArray::getMap;
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java
b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java
index 445d8854a1..ab48c69381 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java
@@ -150,6 +150,8 @@ public interface InternalRow extends DataGetters {
return Timestamp.class;
case ARRAY:
return InternalArray.class;
+ case VECTOR:
+ return InternalVector.class;
case MULTISET:
case MAP:
return InternalMap.class;
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java
new file mode 100644
index 0000000000..3d8416efbf
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.data;
+
+import org.apache.paimon.data.serializer.InternalArraySerializer;
+import org.apache.paimon.io.DataInputViewStreamWrapper;
+import org.apache.paimon.io.DataOutputViewStreamWrapper;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.VectorType;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests that ARRAY<VECTOR> is supported by {@link InternalArray}
accessors: the {@link
+ * InternalArraySerializer} eagerly builds an element getter, so a missing
VECTOR case fails
+ * serializer construction for an accepted DDL type.
+ */
+class InternalArrayVectorGetterTest {
+
+ private static final VectorType VECTOR_TYPE = new VectorType(3, new
FloatType());
+
+ @Test
+ void elementGetterReadsVector() {
+ BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[]
{1.0f, 2.0f, 3.0f});
+ GenericArray array = new GenericArray(new Object[] {vector});
+
+ InternalArray.ElementGetter getter =
InternalArray.createElementGetter(VECTOR_TYPE);
+
+ assertThat(getter.getElementOrNull(array, 0)).isEqualTo(vector);
+ }
+
+ @Test
+ void vectorArrayRoundTripsThroughSerializer() throws Exception {
+ InternalArraySerializer serializer = new
InternalArraySerializer(VECTOR_TYPE);
+ // more than one element, with a null between them: only the binary
form exercises the
+ // 8-byte variable-length slot and the null bit, and the last vector
then starts at an
+ // offset the first one's payload decided
+ GenericArray array =
+ new GenericArray(
+ new Object[] {
+ BinaryVector.fromPrimitiveArray(new float[] {4.0f,
5.0f, 6.0f}),
+ null,
+ BinaryVector.fromPrimitiveArray(new float[] {7.0f,
8.0f, 9.0f})
+ });
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ serializer.serialize(array, new DataOutputViewStreamWrapper(out));
+ InternalArray readBack =
+ serializer.deserialize(
+ new DataInputViewStreamWrapper(
+ new ByteArrayInputStream(out.toByteArray())));
+
+ assertThat(readBack.size()).isEqualTo(3);
+ assertThat(readBack.isNullAt(1)).isTrue();
+ assertThat(readBack.getVector(0).toFloatArray()).isEqualTo(new float[]
{4.0f, 5.0f, 6.0f});
+ assertThat(readBack.getVector(2).toFloatArray()).isEqualTo(new float[]
{7.0f, 8.0f, 9.0f});
+
+ // toObjectArray and copy read the binary form through getDataClass
and the element
+ // getter; the projection and manifest paths use them, no test did
+ InternalVector[] objects = ((BinaryArray)
readBack).toObjectArray(VECTOR_TYPE);
+ assertThat(objects).hasSize(3);
+ assertThat(objects[1]).isNull();
+ assertThat(objects[2].toFloatArray()).isEqualTo(new float[] {7.0f,
8.0f, 9.0f});
+
+ InternalArray copied = serializer.copy(readBack);
+ assertThat(copied.isNullAt(1)).isTrue();
+ assertThat(copied.getVector(2).toFloatArray()).isEqualTo(new float[]
{7.0f, 8.0f, 9.0f});
+ }
+
+ @Test
+ void vectorArrayCopyResolvesDataClass() {
+ // copy() of a GenericArray goes through
InternalRow.getDataClass(VECTOR) to allocate the
+ // element array; without the VECTOR case that throws "Illegal type",
so this pins the
+ // getDataClass fix (the round-trip test above copies raw bytes and
never reaches it).
+ InternalArraySerializer serializer = new
InternalArraySerializer(VECTOR_TYPE);
+ BinaryVector vector = BinaryVector.fromPrimitiveArray(new float[]
{7.0f, 8.0f, 9.0f});
+ GenericArray array = new GenericArray(new Object[] {vector});
+
+ InternalArray copied = serializer.copy(array);
+
+ assertThat(copied.getVector(0).toFloatArray()).isEqualTo(new float[]
{7.0f, 8.0f, 9.0f});
+ }
+}
