liuyehcf opened a new issue, #4020: URL: https://github.com/apache/paimon/issues/4020
### Search before asking - [X] I searched in the [issues](https://github.com/apache/paimon/issues) and found nothing similar. ### Paimon version 0.8.2 ### Compute Engine SDK itself ### Minimal reproduce step ```java package org.byconity.paimon; import org.apache.commons.io.FileUtils; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryArray; import org.apache.paimon.data.BinaryArrayWriter; import org.apache.paimon.data.BinaryWriter; import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.Path; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.table.sink.BatchTableWrite; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.IntType; import org.junit.Test; import java.io.File; import java.util.List; public class ParquetWriteTest { private final DataType intType = new IntType(false); private final DataType innerArrayType = new ArrayType(false, intType); private final DataType outerArrayType = new ArrayType(false, innerArrayType); @Test public void test() throws Exception { String localPath = "/tmp/paimon_warehouse"; FileUtils.deleteDirectory(new File(localPath)); Options options = new Options(); options.set(CatalogOptions.METASTORE, "filesystem"); options.set(CatalogOptions.WAREHOUSE, new Path(localPath).toUri().toString()); CatalogContext context = CatalogContext.create(options); Catalog catalog = CatalogFactory.createCatalog(context); String dbName = "testDb"; String tblName = "testTbl"; catalog.createDatabase(dbName, false); Schema.Builder schemaBuilder = Schema.newBuilder(); schemaBuilder.column("col_nestedarray", outerArrayType); schemaBuilder.option("file.format", "orc"); Schema schema = schemaBuilder.build(); Identifier tableId = Identifier.create(dbName, tblName); catalog.createTable(tableId, schema, false); Table table = catalog.getTable(tableId); BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder().withOverwrite(); try (BatchTableWrite write = writeBuilder.newWrite()) { GenericRow record = GenericRow.of(generateOuterArray()); write.write(record); List<CommitMessage> messages = write.prepareCommit(); try (BatchTableCommit commit = writeBuilder.newCommit()) { commit.commit(messages); } } } private Object generateOuterArray() { BinaryArray binaryArray = new BinaryArray(); BinaryArrayWriter writer = new BinaryArrayWriter(binaryArray, 1, BinaryArray.calculateFixLengthPartSize(innerArrayType)); BinaryWriter.ValueSetter valueSetter = BinaryWriter.createValueSetter(innerArrayType); valueSetter.setValue(writer, 0, generateInnerArray()); writer.complete(); return binaryArray; } private Object generateInnerArray() { BinaryArray binaryArray = new BinaryArray(); BinaryArrayWriter writer = new BinaryArrayWriter(binaryArray, 1, BinaryArray.calculateFixLengthPartSize(intType)); BinaryWriter.ValueSetter valueSetter = BinaryWriter.createValueSetter(intType); valueSetter.setValue(writer, 0, 1); writer.complete(); return binaryArray; } } ``` Same code works fine with orc and avro, but failed with parquet * `schemaBuilder.option("file.format", "orc");` change this line to shift format ### What doesn't meet your expectations? parquet failed to write complex types ### Anything else? Nothing ### Are you willing to submit a PR? - [ ] I'm willing to submit a PR! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
