This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12978-8622bbdc883704faecb14e94bd73313291647438 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit e388eaba95109d5af28fb8f5016b3f2728a3fa9b Author: Goutam Adwant <[email protected]> AuthorDate: Sat Nov 7 06:39:53 2026 +0000 [Improve][Connector-V2] Prune unselected Parquet columns (#12978) Signed-off-by: Goutam Adwant <[email protected]> --- docs/en/connectors/source/LocalFile.md | 4 + docs/zh/connectors/source/LocalFile.md | 3 + .../file/source/reader/ParquetReadStrategy.java | 101 +++- .../source/reader/ParquetReadStrategyTest.java | 532 +++++++++++++++++++++ .../e2e/connector/file/local/LocalFileIT.java | 7 + .../local_file_parquet_projection_to_assert.conf | 9 +- 6 files changed, 639 insertions(+), 17 deletions(-) diff --git a/docs/en/connectors/source/LocalFile.md b/docs/en/connectors/source/LocalFile.md index 59f6a179810..8f6de728ca2 100644 --- a/docs/en/connectors/source/LocalFile.md +++ b/docs/en/connectors/source/LocalFile.md @@ -253,6 +253,10 @@ Note: Only single-column (top-to-bottom) PDF layouts are supported. Multi-column The read column list of the data source, user can use it to implement field projection. +For Parquet files, the reader also skips unselected top-level column data. Selected nested fields +are read in full, and output column order is unchanged. Explicit Hadoop Parquet projections, reader +schemas and record filters retain their existing behavior and disable automatic column pruning. + ### delimiter/field_delimiter [string] **delimiter** parameter will deprecate after version 2.3.5, please use **field_delimiter** instead. diff --git a/docs/zh/connectors/source/LocalFile.md b/docs/zh/connectors/source/LocalFile.md index 84c2c02d22c..7a8683e6012 100644 --- a/docs/zh/connectors/source/LocalFile.md +++ b/docs/zh/connectors/source/LocalFile.md @@ -253,6 +253,9 @@ PDF 特有的解析行为如下: 数据源的读取列列表,用户可以使用它来实现字段投影。 +对于 Parquet 文件,读取器还会跳过未选择的顶层列数据。选中的嵌套字段仍完整读取,输出列顺序保持不变。 +显式配置 Hadoop Parquet 投影、读取模式或记录过滤器时,保持原有行为,不启用自动列裁剪。 + ### delimiter/field_delimiter [string] **delimiter** 参数将在 2.3.5 版本后弃用,请使用 **field_delimiter** 代替。 diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java index 79246a14b5b..d21bdb157fe 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategy.java @@ -42,20 +42,25 @@ import org.apache.avro.data.TimeConversions; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.util.Utf8; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.parquet.HadoopReadOptions; -import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroReadSupport; +import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.NanoTime; import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.api.InitContext; import org.apache.parquet.hadoop.api.ReadSupport; import org.apache.parquet.hadoop.example.GroupReadSupport; import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.api.Binary; +import org.apache.parquet.io.api.RecordMaterializer; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; @@ -78,8 +83,10 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -144,13 +151,19 @@ public class ParquetReadStrategy extends AbstractReadStrategy { final boolean useSplitRange = enableSplitFile && split.getStart() >= 0 && split.getLength() > 0; GenericRecord record; - AvroParquetReader.Builder<GenericData.Record> builder = - AvroParquetReader.<GenericData.Record>builder(hadoopInputFile) - .withDataModel(dataModel); + ParquetReader.Builder<GenericData.Record> builder = + new ProjectedParquetReaderBuilder<>( + hadoopInputFile, + new AvroReadSupport<>(dataModel), + seaTunnelRowType.getFieldNames()); // HadoopConf carries READ_INT96_AS_FIXED / ADD_LIST_ELEMENT_RECORDS. ParquetReader lifts // those automatically only from a HadoopInputFile, so set them explicitly here. withConf() // rebuilds the read options, so it must run before withFileRange() below. - builder.withConf(hadoopFileSystemProxy.getConfiguration()); + Configuration readConfiguration = + new Configuration(hadoopFileSystemProxy.getConfiguration()); + // AvroParquetReader's GenericData builder uses the non-compatible materializer. + readConfiguration.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, false); + builder.withConf(readConfiguration); if (useSplitRange) { long start = split.getStart(); long end = start + split.getLength(); @@ -197,10 +210,12 @@ public class ParquetReadStrategy extends AbstractReadStrategy { // HadoopInputFile.fromStatus), which leaks with the FS cache // disabled. Bind to the connector's FileSystem instead. ParquetReader.Builder<Group> builder = - new GroupParquetReaderBuilder( + new ProjectedParquetReaderBuilder<>( FileSystemInputFile.fromPath( hadoopFileSystemProxy.getFileSystem(), - filePath)); + filePath), + new GroupReadSupport(), + seaTunnelRowType.getFieldNames()); // withConf() rebuilds the read options, so it must precede // withFileRange() below. builder.withConf(hadoopFileSystemProxy.getConfiguration()); @@ -1017,20 +1032,76 @@ public class ParquetReadStrategy extends AbstractReadStrategy { } /** - * Binds the native (Group) Parquet reader to an already-open {@link - * org.apache.hadoop.fs.FileSystem}. {@code ParquetReader.Builder(InputFile)} is protected and - * there is no public {@code builder(ReadSupport, InputFile)}, so this mirrors what {@code - * AvroParquetReader.Builder} does for the Avro path. + * Binds a Parquet reader to an already-open {@link org.apache.hadoop.fs.FileSystem}. {@code + * ParquetReader.Builder(InputFile)} is protected and there is no public {@code + * builder(ReadSupport, InputFile)}, so this mirrors what {@code AvroParquetReader.Builder} does + * for the Avro path. */ - private static final class GroupParquetReaderBuilder extends ParquetReader.Builder<Group> { + private static final class ProjectedParquetReaderBuilder<T> extends ParquetReader.Builder<T> { + private final ReadSupport<T> readSupport; - private GroupParquetReaderBuilder(InputFile file) { + private ProjectedParquetReaderBuilder( + InputFile file, ReadSupport<T> delegate, String[] fields) { super(file); + this.readSupport = new ProjectedReadSupport<>(delegate, fields); } @Override - protected ReadSupport<Group> getReadSupport() { - return new GroupReadSupport(); + protected ReadSupport<T> getReadSupport() { + return readSupport; + } + } + + /** Prunes whole top-level fields while retaining the library's decoding and schema metadata. */ + private static final class ProjectedReadSupport<T> extends ReadSupport<T> { + private final ReadSupport<T> delegate; + private final Set<String> fields; + + private ProjectedReadSupport(ReadSupport<T> delegate, String[] fields) { + this.delegate = delegate; + this.fields = new HashSet<>(Arrays.asList(fields)); + } + + @Override + public ReadContext init(InitContext context) { + ReadContext original = delegate.init(context); + Configuration configuration = context.getConfiguration(); + // Explicit schemas and filters may require columns outside the output schema. + if (configuration.get(ReadSupport.PARQUET_READ_SCHEMA) != null + || configuration.get(AvroReadSupport.AVRO_REQUESTED_PROJECTION) != null + || configuration.get("parquet.avro.read.schema") != null + || configuration.get(ParquetInputFormat.FILTER_PREDICATE) != null + || configuration.get(ParquetInputFormat.UNBOUND_RECORD_FILTER) != null) { + return original; + } + MessageType schema = original.getRequestedSchema(); + List<Type> selected = new ArrayList<>(); + for (Type field : schema.getFields()) { + if (fields.contains(field.getName())) { + selected.add(field); + } + } + // Preserve cardinality/decoding for zero-column and all-missing output schemas. + if (selected.isEmpty() || selected.size() == schema.getFieldCount()) { + return original; + } + if (delegate instanceof AvroReadSupport + && !context.getKeyValueMetadata().containsKey("parquet.avro.schema") + && !context.getKeyValueMetadata().containsKey("avro.schema")) { + // Keep the native fallback for illegal names even in unselected file fields. + new AvroSchemaConverter(configuration).convert(context.getFileSchema()); + } + return new ReadContext( + new MessageType(schema.getName(), selected), original.getReadSupportMetadata()); + } + + @Override + public RecordMaterializer<T> prepareForRead( + Configuration configuration, + Map<String, String> metadata, + MessageType fileSchema, + ReadContext readContext) { + return delegate.prepareForRead(configuration, metadata, fileSchema, readContext); } } } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java index 70ac9f92400..35a96492dae 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ParquetReadStrategyTest.java @@ -31,6 +31,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.file.config.FileBaseSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.file.source.split.FileSourceSplit; +import org.apache.seatunnel.connectors.seatunnel.file.source.split.ParquetFileSplitStrategy; import org.apache.seatunnel.connectors.seatunnel.file.util.LocalFileSystemConf; import org.apache.avro.Conversions; @@ -41,24 +43,38 @@ import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroup; +import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.api.ReadSupport; import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.hadoop.example.GroupWriteSupport; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.io.DelegatingPositionOutputStream; +import org.apache.parquet.io.OutputFile; +import org.apache.parquet.io.PositionOutputStream; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.apache.parquet.schema.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import lombok.extern.slf4j.Slf4j; @@ -67,15 +83,22 @@ import java.io.IOException; import java.math.BigDecimal; import java.net.URL; import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicLong; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; @@ -83,6 +106,515 @@ import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; @Slf4j public class ParquetReadStrategyTest { + @TempDir java.nio.file.Path projectionDirectory; + + @Test + void testProjectionFixtureOutputPreservesPositionAndCreateMode() throws Exception { + Path path = new Path(projectionDirectory.resolve("output.parquet").toUri()); + OutputFile file = new ProjectionOutputFile(path); + try (PositionOutputStream output = file.create(0)) { + output.write(1); + output.write(new byte[] {2, 3, 4}, 0, 2); + Assertions.assertEquals(3, output.getPos()); + } + Assertions.assertThrows(FileAlreadyExistsException.class, () -> file.create(0)); + try (PositionOutputStream output = file.createOrOverwrite(0)) { + Assertions.assertEquals(0, output.getPos()); + output.write(new byte[] {5, 6}); + Assertions.assertEquals(2, output.getPos()); + } + Assertions.assertArrayEquals( + new byte[] {5, 6}, Files.readAllBytes(Paths.get(path.toUri()))); + } + + /** Writes projection fixtures without Hadoop's platform-specific local permission helpers. */ + private static class ProjectionOutputFile implements OutputFile { + private final java.nio.file.Path path; + + private ProjectionOutputFile(Path path) { + this.path = Paths.get(path.toUri()); + } + + @Override + public PositionOutputStream create(long blockSizeHint) throws IOException { + return open(StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + } + + @Override + public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException { + return open( + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } + + private PositionOutputStream open(StandardOpenOption... options) throws IOException { + FileChannel channel = FileChannel.open(path, options); + return new DelegatingPositionOutputStream(Channels.newOutputStream(channel)) { + @Override + public long getPos() throws IOException { + return channel.position(); + } + }; + } + + @Override + public boolean supportsBlockSize() { + return false; + } + + @Override + public long defaultBlockSize() { + return 0; + } + } + + @Test + void testProjectionRetainsAvroSchemaMetadataAndEmptyInput() throws Exception { + Schema schema = + new Schema.Parser() + .parse( + "{\"type\":\"record\",\"name\":\"Named\"," + + "\"namespace\":\"example\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"}," + + "{\"name\":\"text\",\"type\":\"string\",\"aliases\":[\"old_text\"]}," + + "{\"name\":\"unused\",\"type\":[\"null\",\"string\"],\"default\":null}]}"); + for (boolean empty : new boolean[] {false, true}) { + Path file = new Path(projectionDirectory.resolve("avro-" + empty + ".parquet").toUri()); + try (ParquetWriter<GenericData.Record> writer = + AvroParquetWriter.<GenericData.Record>builder(new ProjectionOutputFile(file)) + .withSchema(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + if (!empty) { + GenericData.Record record = new GenericData.Record(schema); + record.put("id", 7); + record.put("text", "value"); + record.put("unused", null); + writer.write(record); + } + } + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(new LocalFileSystemConf.LocalConf("file:///")); + strategy.setPluginConfig( + ConfigFactory.parseString( + "read_columns=[text,id]\nparse_partition_from_path=false")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + TestCollector collector = new TestCollector(); + strategy.read(file.toString(), "", collector); + Assertions.assertEquals(empty ? 0 : 1, collector.getRows().size()); + if (!empty) { + Assertions.assertArrayEquals( + new Object[] {"value", 7}, collector.getRows().get(0).getFields()); + } + } finally { + strategy.close(); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testProjectionRetainsLogicalSplitBoundaries(boolean nativeReader) throws Exception { + String id = nativeReader ? "id-value" : "id"; + MessageType schema = + Types.buildMessage() + .required(INT32) + .named(id) + .required(BINARY) + .named("unused") + .named("split"); + Path file = new Path(projectionDirectory.resolve("split.parquet").toUri()); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(file)) + .withType(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withDictionaryEncoding(false) + .withRowGroupSize(1024) + .withMinRowCountForPageSizeCheck(1) + .withMaxRowCountForPageSizeCheck(1) + .build()) { + for (int i = 0; i < 40; i++) { + writer.write( + new SimpleGroup(schema).append(id, i).append("unused", "x".repeat(1024))); + } + } + List<FileSourceSplit> splits; + try (ParquetFileReader fileReader = + ParquetFileReader.open(HadoopInputFile.fromPath(file, new Configuration()))) { + Assertions.assertTrue(fileReader.getFooter().getBlocks().size() > 1); + splits = new ParquetFileSplitStrategy(1).split("table", file.toString()); + } + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(new LocalFileSystemConf.LocalConf("file:///")); + strategy.setPluginConfig( + ConfigFactory.parseString( + "read_columns=[\"" + + id + + "\"]\nparse_partition_from_path=false\nenable_file_split=true")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + TestCollector collector = new TestCollector(); + for (FileSourceSplit split : splits) { + strategy.read(split, collector); + } + Assertions.assertEquals(40, collector.getRows().size()); + for (int i = 0; i < 40; i++) { + Assertions.assertEquals(i, collector.getRows().get(i).getField(0)); + Assertions.assertEquals("table", collector.getRows().get(i).getTableId()); + } + } finally { + strategy.close(); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "parquet.avro.projection", + "parquet.avro.read.schema", + "parquet.read.schema" + }) + void testProjectionPreservesExplicitReaderSchemas(String option) throws Exception { + boolean nativeReader = option.equals(ReadSupport.PARQUET_READ_SCHEMA); + String id = nativeReader ? "id-value" : "id"; + MessageType schema = + Types.buildMessage() + .required(INT32) + .named(id) + .required(INT32) + .named("unused") + .named("explicit"); + Path file = new Path(projectionDirectory.resolve("explicit.parquet").toUri()); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(file)) + .withType(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + writer.write(new SimpleGroup(schema).append(id, 1).append("unused", 2)); + } + String explicit = + nativeReader + ? "message explicit { required int32 unused; }" + : "{\"type\":\"record\",\"name\":\"explicit\",\"fields\":[{\"name\":\"unused\",\"type\":\"int\"}]}"; + boolean readSchema = option.equals("parquet.avro.read.schema"); + if (readSchema) { + explicit = + "{\"type\":\"record\",\"name\":\"explicit\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"},{\"name\":\"unused\",\"type\":\"int\"}]}"; + } + LocalFileSystemConf.LocalConf conf = new LocalFileSystemConf.LocalConf("file:///"); + conf.getExtraOptions().put(option, explicit); + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(conf); + strategy.setPluginConfig( + ConfigFactory.parseString( + "read_columns=[\"" + id + "\"]\nparse_partition_from_path=false")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + TestCollector collector = new TestCollector(); + strategy.read(file.toString(), "", collector); + Assertions.assertEquals(1, collector.getRows().size()); + Assertions.assertEquals(readSchema ? 1 : null, collector.getRows().get(0).getField(0)); + Assertions.assertEquals( + explicit, strategy.hadoopFileSystemProxy.getConfiguration().get(option)); + } finally { + strategy.close(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testProjectionPreservesFiltersOnUnselectedColumns(boolean nativeReader) throws Exception { + String id = nativeReader ? "id-value" : "id"; + MessageType schema = + Types.buildMessage() + .required(INT32) + .named(id) + .required(INT32) + .named("filter_column") + .named("filtered"); + Path file = new Path(projectionDirectory.resolve("filter.parquet").toUri()); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(file)) + .withType(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + writer.write(new SimpleGroup(schema).append(id, 1).append("filter_column", 0)); + writer.write(new SimpleGroup(schema).append(id, 2).append("filter_column", 1)); + } + Configuration filterConfiguration = new Configuration(false); + ParquetInputFormat.setFilterPredicate( + filterConfiguration, FilterApi.eq(FilterApi.intColumn("filter_column"), 1)); + LocalFileSystemConf.LocalConf conf = new LocalFileSystemConf.LocalConf("file:///"); + conf.getExtraOptions() + .put( + ParquetInputFormat.FILTER_PREDICATE, + filterConfiguration.get(ParquetInputFormat.FILTER_PREDICATE)); + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(conf); + strategy.setPluginConfig( + ConfigFactory.parseString( + "read_columns=[\"" + id + "\"]\nparse_partition_from_path=false")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + TestCollector collector = new TestCollector(); + strategy.read(file.toString(), "", collector); + Assertions.assertEquals(1, collector.getRows().size()); + Assertions.assertEquals(2, collector.getRows().get(0).getField(0)); + } finally { + strategy.close(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testProjectionRetainsNestedLogicalNullAndPartitionValues(boolean nativeReader) + throws Exception { + String unused = nativeReader ? "unused-value" : "unused"; + MessageType schema = + MessageTypeParser.parseMessageType( + "message row { required int32 id; " + + "optional binary nullable (UTF8); required int32 day (DATE); " + + "required group nested { required int32 number; required binary text (UTF8); } " + + "required binary " + + unused + + "; }"); + java.nio.file.Path directory = + Files.createDirectories(projectionDirectory.resolve("region=east")); + Path file = new Path(directory.resolve("nested.parquet").toUri()); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(file)) + .withType(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + SimpleGroup record = new SimpleGroup(schema); + record.append("id", 7).append("day", 1).append(unused, "discard"); + record.addGroup("nested").append("number", 3).append("text", "value"); + writer.write(record); + } + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(new LocalFileSystemConf.LocalConf("file:///")); + strategy.setPluginConfig( + ConfigFactory.parseString("read_columns = [nested, day, nullable, id]")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + TestCollector collector = new TestCollector(); + strategy.read(new FileSourceSplit("table", file.toString()), collector); + Assertions.assertEquals(1, collector.getRows().size()); + SeaTunnelRow row = collector.getRows().get(0); + Assertions.assertEquals("table", row.getTableId()); + Assertions.assertEquals(5, row.getArity()); + Assertions.assertArrayEquals( + new Object[] {3, "value"}, ((SeaTunnelRow) row.getField(0)).getFields()); + Assertions.assertEquals(LocalDate.ofEpochDay(1), row.getField(1)); + Assertions.assertNull(row.getField(2)); + Assertions.assertEquals(7, row.getField(3)); + Assertions.assertEquals("east", row.getField(4)); + } finally { + strategy.close(); + } + } + + @Test + void testProjectionIsPerFileAndPreservesMissingColumns() throws Exception { + MessageType firstSchema = + MessageTypeParser.parseMessageType( + "message first { required int32 id; required int32 first_only; }"); + MessageType secondSchema = + MessageTypeParser.parseMessageType( + "message second { required int32 second_only; required int32 id; }"); + Path first = new Path(projectionDirectory.resolve("first.parquet").toUri()); + Path second = new Path(projectionDirectory.resolve("second.parquet").toUri()); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(first)) + .withType(firstSchema) + .build()) { + writer.write(new SimpleGroup(firstSchema).append("id", 1).append("first_only", 9)); + } + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(second)) + .withType(secondSchema) + .build()) { + writer.write(new SimpleGroup(secondSchema).append("id", 2).append("second_only", 8)); + } + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(new LocalFileSystemConf.LocalConf("file:///")); + strategy.setPluginConfig(ConfigFactory.parseString("parse_partition_from_path=false")); + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"first_only", "id", "absent"}, + new SeaTunnelDataType[] { + BasicType.INT_TYPE, BasicType.INT_TYPE, BasicType.INT_TYPE + }); + strategy.setCatalogTable(CatalogTableUtil.getCatalogTable("table", rowType)); + Configuration shared = strategy.hadoopFileSystemProxy.getConfiguration(); + TestCollector collector = new TestCollector(); + strategy.read(first.toString(), "", collector); + strategy.read(second.toString(), "", collector); + Assertions.assertArrayEquals( + new Object[] {9, 1, null}, collector.getRows().get(0).getFields()); + Assertions.assertArrayEquals( + new Object[] {null, 2, null}, collector.getRows().get(1).getFields()); + strategy.setCatalogTable( + CatalogTableUtil.getCatalogTable( + "table", + new SeaTunnelRowType( + new String[] {"absent"}, + new SeaTunnelDataType[] {BasicType.INT_TYPE}))); + strategy.read(second.toString(), "", collector); + Assertions.assertArrayEquals( + new Object[] {null}, collector.getRows().get(2).getFields()); + Assertions.assertNull(shared.get(ReadSupport.PARQUET_READ_SCHEMA)); + Assertions.assertNull(shared.get(AvroReadSupport.AVRO_REQUESTED_PROJECTION)); + Assertions.assertNull(shared.get(AvroReadSupport.AVRO_COMPATIBILITY)); + } finally { + strategy.close(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testReadColumnsPrunesPhysicalColumns(boolean nativeReader) throws Exception { + String id = nativeReader ? "id-value" : "id"; + Path file = new Path(projectionDirectory.resolve("wide.parquet").toUri()); + MessageType schema = + Types.buildMessage() + .required(INT32) + .named(id) + .required(BINARY) + .named("unused") + .named("wide"); + Random random = new Random(1234); + try (ParquetWriter<Group> writer = + ExampleParquetWriter.builder(new ProjectionOutputFile(file)) + .withType(schema) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .withDictionaryEncoding(false) + .build()) { + for (int i = 0; i < 128; i++) { + byte[] bytes = new byte[32768]; + random.nextBytes(bytes); + writer.write( + new SimpleGroup(schema) + .append(id, i) + .append( + "unused", + org.apache.parquet.io.api.Binary.fromConstantByteArray( + bytes))); + } + } + LocalFileSystemConf.LocalConf conf = + new LocalFileSystemConf.LocalConf("file:///") { + @Override + public String getFsHdfsImpl() { + return CountingFileSystem.class.getName(); + } + }; + ParquetReadStrategy strategy = new ParquetReadStrategy(); + try { + strategy.init(conf); + strategy.setPluginConfig( + ConfigFactory.parseString( + "read_columns = [\"" + id + "\"]\nparse_partition_from_path = false")); + strategy.getSeaTunnelRowTypeInfo(file.toString()); + CountingFileSystem.BYTES.set(0); + TestCollector collector = new TestCollector(); + strategy.read(file.toString(), "", collector); + long bytes = CountingFileSystem.BYTES.get(); + Assertions.assertEquals(128, collector.getRows().size()); + for (int i = 0; i < 128; i++) { + Assertions.assertEquals(1, collector.getRows().get(i).getArity()); + Assertions.assertEquals(i, collector.getRows().get(i).getField(0)); + } + long fileBytes = Files.size(projectionDirectory.resolve("wide.parquet")); + System.out.printf( + "PARQUET_PROJECTION native=%s rows=128 checksum=8128 file_bytes=%d read_bytes=%d%n", + nativeReader, fileBytes, bytes); + ParquetReadStrategy allColumns = new ParquetReadStrategy(); + try { + allColumns.init(conf); + allColumns.setPluginConfig( + ConfigFactory.parseString("parse_partition_from_path=false")); + allColumns.getSeaTunnelRowTypeInfo(file.toString()); + CountingFileSystem.BYTES.set(0); + TestCollector allCollector = new TestCollector(); + allColumns.read(file.toString(), "", allCollector); + long fullBytes = CountingFileSystem.BYTES.get(); + Assertions.assertEquals(128, allCollector.getRows().size()); + Random expected = new Random(1234); + for (int i = 0; i < 128; i++) { + SeaTunnelRow row = allCollector.getRows().get(i); + byte[] payload = new byte[32768]; + expected.nextBytes(payload); + Assertions.assertEquals(2, row.getArity()); + Assertions.assertEquals(i, row.getField(0)); + Assertions.assertArrayEquals(payload, (byte[]) row.getField(1)); + } + Assertions.assertTrue(fullBytes > fileBytes / 2); + System.out.printf( + "PARQUET_ALL_COLUMNS native=%s rows=128 checksum=8128 read_bytes=%d%n", + nativeReader, fullBytes); + } finally { + allColumns.close(); + } + Assertions.assertTrue( + bytes < fileBytes / 4, + "Unselected Parquet column was read: " + bytes + " of " + fileBytes); + } finally { + strategy.close(); + } + } + + public static class CountingFileSystem extends RawLocalFileSystem { + static final AtomicLong BYTES = new AtomicLong(); + + @Override + public FSDataInputStream open(Path path, int bufferSize) throws IOException { + FSDataInputStream input = super.open(path, bufferSize); + return new FSDataInputStream( + new FSInputStream() { + @Override + public void seek(long position) throws IOException { + input.seek(position); + } + + @Override + public long getPos() throws IOException { + return input.getPos(); + } + + @Override + public boolean seekToNewSource(long position) throws IOException { + return input.seekToNewSource(position); + } + + @Override + public int read() throws IOException { + int value = input.read(); + if (value >= 0) { + BYTES.incrementAndGet(); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int read = input.read(bytes, offset, length); + if (read > 0) { + BYTES.addAndGet(read); + } + return read; + } + + @Override + public void close() throws IOException { + input.close(); + } + }); + } + } + @Test public void testParquetRead1() throws Exception { URL resource = ParquetReadStrategyTest.class.getResource("/timestamp_as_int64.parquet"); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index 97a7462fa0b..a8aa98c614d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -441,6 +441,13 @@ public class LocalFileIT extends TestSuiteBase { helper.execute("/json/local_file_to_console.conf"); } + @TestTemplate + public void testLocalFileParquetProjection(TestContainer container) + throws IOException, InterruptedException { + TestHelper helper = new TestHelper(container); + helper.execute("/parquet/local_file_parquet_projection_to_assert.conf"); + } + @TestTemplate public void testLocalFileOrcParquetBinaryXml(TestContainer container) throws IOException, InterruptedException { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf index 871b3c02576..de3d2add530 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/parquet/local_file_parquet_projection_to_assert.conf @@ -31,7 +31,8 @@ source { LocalFile { path = "/seatunnel/read/parquet" file_format_type = "parquet" - read_columns = [c_string, c_boolean, c_double] + # Keep output order independent of the physical Parquet column order. + read_columns = [c_double, c_boolean, c_string] plugin_output = "fake" } } @@ -40,6 +41,10 @@ sink { Assert { rules { row_rules = [ + { + rule_type = MIN_ROW + rule_value = 5 + }, { rule_type = MAX_ROW rule_value = 5 @@ -76,4 +81,4 @@ sink { ] } } -} \ No newline at end of file +}
