This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 90bb27c6ff [avro] Fix ClassCastException when writing MULTISET with
non-string element type (#9066)
90bb27c6ff is described below
commit 90bb27c6ff5b2fb6b21cd5f49572f07a97fb4315
Author: Eunbin Son <[email protected]>
AuthorDate: Thu Aug 6 23:15:49 2026 +0900
[avro] Fix ClassCastException when writing MULTISET with non-string element
type (#9066)
---
docs/docs/concepts/spec/fileformat.md | 16 +++---
.../paimon/format/avro/AvroSchemaVisitor.java | 13 ++++-
.../format/avro/AvroFormatReadWriteTest.java | 67 ++++++++++++++++++++++
3 files changed, 85 insertions(+), 11 deletions(-)
diff --git a/docs/docs/concepts/spec/fileformat.md
b/docs/docs/concepts/spec/fileformat.md
index 541498df57..2d199e6f15 100644
--- a/docs/docs/concepts/spec/fileformat.md
+++ b/docs/docs/concepts/spec/fileformat.md
@@ -234,16 +234,16 @@ The following table lists the type mapping from Paimon
type to Avro type.
<td></td>
</tr>
<tr>
- <td><code>MAP</code><br>
- (key must be string/char/varchar type)</td>
- <td><code>map</code></td>
- <td></td>
+ <td><code>MAP</code></td>
+ <td>string/char/varchar key: <code>map</code><br>
+ other key: <code>array</code> of key-value <code>record</code></td>
+ <td>other key: <code>map</code></td>
</tr>
<tr>
- <td><code>MULTISET</code><br>
- (element must be string/char/varchar type)</td>
- <td><code>map</code></td>
- <td></td>
+ <td><code>MULTISET</code></td>
+ <td>string/char/varchar element: <code>map</code><br>
+ other element: <code>array</code> of element-count
<code>record</code></td>
+ <td>other element: <code>map</code></td>
</tr>
<tr>
<td><code>ROW</code></td>
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaVisitor.java
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaVisitor.java
index aa329c6132..3ffe8bfeae 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaVisitor.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroSchemaVisitor.java
@@ -23,6 +23,7 @@ import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VectorType;
@@ -51,9 +52,15 @@ public interface AvroSchemaVisitor<T> {
return visitUnion(schema, type);
case ARRAY:
- if (type instanceof MapType) {
- MapType mapType = (MapType) type;
- return visitArrayMap(schema, mapType.getKeyType(),
mapType.getValueType());
+ if (type instanceof MapType || type instanceof MultisetType) {
+ // A multiset is encoded as a map from element to its
multiplicity, so it uses
+ // the same array-of-record encoding as a map with a
non-string key. Reuse the
+ // converter's key/value extraction so that schema
creation and schema visiting
+ // always agree on that encoding.
+ return visitArrayMap(
+ schema,
+ AvroSchemaConverter.extractKeyTypeToAvroMap(type),
+
AvroSchemaConverter.extractValueTypeToAvroMap(type));
} else if (type instanceof VectorType) {
return visitArrayVector(schema, ((VectorType)
type).getElementType());
} else {
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFormatReadWriteTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFormatReadWriteTest.java
index 4a7d724635..43228b9487 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFormatReadWriteTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFormatReadWriteTest.java
@@ -18,20 +18,34 @@
package org.apache.paimon.format.avro;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.BinaryVector;
+import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FileFormatFactory;
import org.apache.paimon.format.FormatReadWriteTest;
+import org.apache.paimon.format.FormatReaderContext;
import org.apache.paimon.options.Options;
+import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
/** An avro {@link FormatReadWriteTest}. */
public class AvroFormatReadWriteTest extends FormatReadWriteTest {
@@ -50,6 +64,59 @@ public class AvroFormatReadWriteTest extends
FormatReadWriteTest {
testArrayBlobDescriptorRoundTrip();
}
+ @Test
+ public void testMultisetWithNonStringElement() throws IOException {
+ // a non-string element makes the multiset use the array-of-record
encoding
+ testMultisetRoundTrip(DataTypes.INT().notNull(), 10, 20);
+ }
+
+ @Test
+ public void testMultisetWithStringElement() throws IOException {
+ // a string element makes the multiset use the native avro map encoding
+ testMultisetRoundTrip(
+ DataTypes.STRING().notNull(),
+ BinaryString.fromString("a"),
+ BinaryString.fromString("b"));
+ }
+
+ private void testMultisetRoundTrip(DataType elementType, Object first,
Object second)
+ throws IOException {
+ RowType rowType =
+ RowType.builder()
+ .field("id", DataTypes.INT().notNull())
+ .field("ms", DataTypes.MULTISET(elementType))
+ .build();
+ Map<Object, Object> multiset = new HashMap<>();
+ multiset.put(first, 1);
+ multiset.put(second, 2);
+ GenericRow expected = GenericRow.of(1, new GenericMap(multiset));
+
+ FileFormat format = fileFormat();
+ write(format.createWriterFactory(rowType), file, expected);
+
+ List<InternalRow> result = new ArrayList<>();
+ try (RecordReader<InternalRow> reader =
+ format.createReaderFactory(rowType, rowType, new ArrayList<>())
+ .createReader(
+ new FormatReaderContext(fileIO, file,
fileIO.getFileSize(file)))) {
+ InternalRowSerializer serializer = new
InternalRowSerializer(rowType);
+ reader.forEachRemaining(row -> result.add(serializer.copy(row)));
+ }
+
+ assertThat(result).hasSize(1);
+ assertThat(result.get(0).getInt(0)).isEqualTo(1);
+ InternalMap actual = result.get(0).getMap(1);
+ assertThat(actual.size()).isEqualTo(multiset.size());
+ InternalArray.ElementGetter keyGetter =
InternalArray.createElementGetter(elementType);
+ InternalArray keys = actual.keyArray();
+ InternalArray values = actual.valueArray();
+ Map<Object, Object> actualMultiset = new HashMap<>();
+ for (int i = 0; i < actual.size(); i++) {
+ actualMultiset.put(keyGetter.getElementOrNull(keys, i),
values.getInt(i));
+ }
+ assertThat(actualMultiset).isEqualTo(multiset);
+ }
+
@Override
protected RowType rowTypeForFullTypesTest() {
RowType rowWithoutVector = super.rowTypeForFullTypesTest();
