This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20654-88b20661a16ab228b312adf94faad909debbe9ce in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d020350d0d0605b7e30d2bcde42de39067529b13 Author: Shuo Cheng <[email protected]> AuthorDate: Mon Nov 16 04:14:29 2026 +0000 fix(flink): filter pushdown predicates by physical Parquet schema (#20654) * fix(flink): skip predicate pushdown for native delete logs * fix(flink): validate predicates against physical Parquet schema * fix(flink): validate configured Parquet predicates Check configured predicate columns against each file's physical schema. Remove unsupported predicates from a per-file configuration copy so ParquetFileReader cannot reload them, while preserving the shared configuration and valid filters for other files. Add native-log and snapshot regressions for configured filters, compound predicates, and configuration isolation. --- .../format/ParquetPredicateColumnValidator.java | 113 ++++++++++ .../apache/hudi/table/format/RecordIterators.java | 48 ++-- .../format/TestFlinkRowDataReaderContext.java | 247 +++++++++++++++++++++ .../hudi/table/format/TestRecordIterators.java | 140 +++++++++++- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- .../table/format/cow/ParquetSplitReaderUtil.java | 7 +- .../reader/ParquetColumnarRowSplitReader.java | 6 +- .../format/cow/TestParquetSplitReaderUtil.java | 7 +- 22 files changed, 577 insertions(+), 91 deletions(-) diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java new file mode 100644 index 000000000000..1541be0f4628 --- /dev/null +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/ParquetPredicateColumnValidator.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.table.format; + +import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.filter2.predicate.Operators; +import org.apache.parquet.filter2.predicate.UserDefinedPredicate; +import org.apache.parquet.schema.MessageType; + +/** Checks physical column availability without evaluating or rewriting a Parquet predicate. */ +final class ParquetPredicateColumnValidator implements FilterPredicate.Visitor<Boolean> { + private final MessageType fileSchema; + + private ParquetPredicateColumnValidator(MessageType fileSchema) { + this.fileSchema = fileSchema; + } + + static boolean hasAllColumns(FilterPredicate predicate, MessageType fileSchema) { + try { + return predicate.accept(new ParquetPredicateColumnValidator(fileSchema)); + } catch (UnsupportedOperationException e) { + // Newer Parquet versions can add operators absent from the minimum supported API. + // Omit the whole predicate if its column references cannot be checked. + return false; + } + } + + private boolean contains(Operators.Column<?> column) { + return fileSchema.containsPath(column.getColumnPath().toArray()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.Eq<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.NotEq<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.Lt<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.LtEq<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.Gt<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.GtEq<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.In<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>> Boolean visit(Operators.NotIn<T> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public Boolean visit(Operators.And predicate) { + return predicate.getLeft().accept(this) && predicate.getRight().accept(this); + } + + @Override + public Boolean visit(Operators.Or predicate) { + return predicate.getLeft().accept(this) && predicate.getRight().accept(this); + } + + @Override + public Boolean visit(Operators.Not predicate) { + return predicate.getPredicate().accept(this); + } + + @Override + public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean visit(Operators.UserDefined<T, U> predicate) { + return contains(predicate.getColumn()); + } + + @Override + public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean visit(Operators.LogicalNotUserDefined<T, U> predicate) { + return predicate.getUserDefined().accept(this); + } +} diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java index 67665e1b17a6..9e62c1e4a9ef 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java @@ -40,8 +40,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.hadoop.BadConfigurationException; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.ConfigurationUtil; import org.apache.parquet.hadoop.util.SerializationUtil; +import org.apache.parquet.schema.MessageType; import java.io.IOException; import java.util.LinkedHashMap; @@ -49,6 +51,8 @@ import java.util.List; import java.util.Map; import static org.apache.parquet.filter2.predicate.FilterApi.and; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.apache.parquet.hadoop.ParquetInputFormat.FILTER_PREDICATE; import static org.apache.parquet.hadoop.ParquetInputFormat.UNBOUND_RECORD_FILTER; @@ -105,14 +109,19 @@ public abstract class RecordIterators { long splitStart, long splitLength, List<Predicate> predicates) throws IOException { - FilterPredicate filterPredicate = getFilterPredicate(conf); - for (Predicate predicate : predicates) { - FilterPredicate filter = predicate.filter(); - if (filter != null) { - filterPredicate = filterPredicate == null ? filter : and(filterPredicate, filter); - } + // Read the footer once for both predicate validation and the split reader. + ParquetMetadata footer = readFooter(conf, new org.apache.hadoop.fs.Path(path.toUri()), range(splitStart, splitStart + splitLength)); + MessageType fileSchema = footer.getFileMetaData().getSchema(); + FilterPredicate configuredFilter = getFilterPredicate(conf); + Configuration readConf = conf; + if (configuredFilter != null && !ParquetPredicateColumnValidator.hasAllColumns(configuredFilter, fileSchema)) { + // ParquetFileReader reloads the configured predicate, so remove it from a per-file copy as well. + readConf = new Configuration(conf); + readConf.unset(FILTER_PREDICATE); + configuredFilter = null; } - UnboundRecordFilter recordFilter = getUnboundRecordFilterInstance(conf); + FilterPredicate filterPredicate = createFilterPredicate(fileSchema, predicates, configuredFilter); + UnboundRecordFilter recordFilter = getUnboundRecordFilterInstance(readConf); InternalSchema mergeSchema = internalSchemaManager.getMergeSchema(getFileName(path)); if (mergeSchema.isEmptySchema()) { @@ -120,15 +129,14 @@ public abstract class RecordIterators { ParquetSplitReaderUtil.genPartColumnarRowReader( utcTimestamp, caseSensitive, - conf, + readConf, fieldNames, fieldTypes, partitionSpec, selectedFields, batchSize, path, - splitStart, - splitLength, + footer, filterPredicate, recordFilter)); } else { @@ -138,15 +146,14 @@ public abstract class RecordIterators { ParquetSplitReaderUtil.genPartColumnarRowReader( utcTimestamp, caseSensitive, - conf, + readConf, internalSchemaManager.getMergeFieldNames(mergeSchema, fieldNames), // the reconciled field names castMap.getFileFieldTypes(), // the reconciled field types partitionSpec, selectedFields, batchSize, path, - splitStart, - splitLength, + footer, filterPredicate, recordFilter)); if (castProjection.isPresent()) { @@ -157,6 +164,21 @@ public abstract class RecordIterators { } } + static FilterPredicate createFilterPredicate(MessageType fileSchema, List<Predicate> predicates, FilterPredicate configuredFilter) { + FilterPredicate filterPredicate = configuredFilter; + for (Predicate predicate : predicates) { + // Missing columns are null to Parquet, which can incorrectly prune native delete records. + // Skip the whole predicate if any reference is absent; pruning only one branch of OR/NOT is unsafe. + if (predicate.references().stream().allMatch(column -> fileSchema.containsPath(column.split("\\.")))) { + FilterPredicate filter = predicate.filter(); + if (filter != null) { + filterPredicate = filterPredicate == null ? filter : and(filterPredicate, filter); + } + } + } + return filterPredicate; + } + private static String getFileName(Path path) { if (InLineFSUtils.SCHEME.equals(path.toUri().getScheme())) { return InLineFSUtils.getOuterFilePathFromInlinePath(new StoragePath(path.toUri())).getName(); diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java index f89b367441a4..54f44a626030 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestFlinkRowDataReaderContext.java @@ -19,28 +19,68 @@ package org.apache.hudi.table.format; +import org.apache.hudi.common.config.RecordMergeMode; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.model.HoodieBaseFile; import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.schema.HoodieSchemaType; +import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.table.read.BufferedRecord; +import org.apache.hudi.common.table.read.HoodieFileGroupReader; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.source.ExpressionPredicates; +import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; +import org.apache.avro.generic.IndexedRecord; +import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; +import org.apache.flink.table.expressions.FieldReferenceExpression; +import org.apache.flink.table.expressions.ValueLiteralExpression; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.filter2.predicate.FilterApi; +import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.ParquetInputFormat; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.HOODIE_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_WRITE_TOKEN; +import static org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils.DATA_GEN; +import static org.apache.hudi.common.testutils.reader.HoodieFileSliceTestUtils.ROW_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -132,6 +172,213 @@ class TestFlinkRowDataReaderContext { assertTrue(result.getBoolean(2)); } + @Nested + class NativeLogFilters { + private static final String PARTITION = "partition"; + private static final String FILE_ID = "file-id"; + + @TempDir + Path tempDir; + + private HoodieTableMetaClient metaClient; + private HoodieStorage storage; + private StoragePath partitionPath; + + @BeforeEach + void setUpNativeLogs() throws Exception { + Properties properties = new Properties(); + properties.setProperty(HoodieTableConfig.VERSION.key(), "10"); + properties.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + properties.setProperty(HoodieTableConfig.RECORDKEY_FIELDS.key(), ROW_KEY); + properties.setProperty(HoodieTableConfig.RECORD_MERGE_MODE.key(), RecordMergeMode.EVENT_TIME_ORDERING.name()); + properties.setProperty(HoodieTableConfig.ORDERING_FIELDS.key(), "timestamp"); + metaClient = HoodieTestUtils.init(new HadoopStorageConfiguration(new Configuration()), tempDir.toString(), + HoodieTableType.MERGE_ON_READ, properties); + assertEquals(HoodieTableVersion.TEN, metaClient.getTableConfig().getTableVersion()); + storage = metaClient.getStorage(); + partitionPath = new StoragePath(metaClient.getBasePath(), PARTITION); + storage.createDirectory(partitionPath); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testPrimaryKeyFilterDoesNotResurrectDeletedRecord(boolean configuredFilter) throws Exception { + HoodieBaseFile baseFile = HoodieFileSliceTestUtils.createBaseFile( + new StoragePath(partitionPath, FSUtils.makeBaseFileName("001", DEFAULT_WRITE_TOKEN, FILE_ID, ".parquet")).toString(), + Arrays.asList(record("k", 100L), record("survivor", 100L)), HOODIE_SCHEMA, "001"); + HoodieLogFile deleteFile = createDeleteFile(); + assertTrue(FSUtils.isNativeDeleteLogFile(deleteFile.getPath().getName())); + // Establish that the physical delete schema does not contain the business key column. + try (HoodieRowDataParquetReader reader = new HoodieRowDataParquetReader(storage, deleteFile.getPath())) { + assertEquals(Arrays.asList("_hoodie_record_key", "timestamp"), + reader.getSchema().getFields().stream().map(HoodieSchemaField::name).collect(Collectors.toList())); + } + assertEquals(Collections.singletonList("survivor"), readSnapshot(baseFile, deleteFile, null)); + assertEquals(Collections.singletonList("survivor"), readSnapshot(baseFile, deleteFile, "survivor")); + if (configuredFilter) { + ParquetInputFormat.setFilterPredicate(storage.getConf().unwrapAs(Configuration.class), keyEquals(ROW_KEY, "k").filter()); + } + assertEquals(Collections.emptyList(), readSnapshot(baseFile, deleteFile, "k")); + } + + @Test + void testStoredRecordKeyPredicatePrunesDeleteRowGroups() throws Exception { + HoodieLogFile deleteFile = createDeleteFile(); + assertEquals(Collections.singletonList("k"), readDeleteKeys(deleteFile, keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k"))); + assertEquals(Collections.emptyList(), readDeleteKeys(deleteFile, keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other"))); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("org.apache.hudi.table.format.TestFlinkRowDataReaderContext#deleteLogPredicates") + void testDeleteLogPredicateSafety(ExpressionPredicates.Predicate predicate, List<String> expectedKeys) throws Exception { + assertEquals(expectedKeys, readDeleteKeys(createDeleteFile(), predicate)); + } + + @Test + void testNativeDataLogRetainsPrimaryKeyPushdown() throws Exception { + HoodieLogFile dataFile = HoodieFileSliceTestUtils.createNativeDataLogFile( + nativeLogPath(".log.parquet"), Collections.singletonList(record("other", 100L)), HOODIE_SCHEMA, "002"); + FlinkRowDataReaderContext context = readerContext("k"); + context.setHasLogFiles(true); + try (ClosableIterator<RowData> iterator = context.getFileRecordIterator(dataFile.getPath(), 0, Long.MAX_VALUE, + HOODIE_SCHEMA, HOODIE_SCHEMA, storage)) { + assertFalse(iterator.hasNext(), "Primary key pushdown should still prune nonmatching data log row groups"); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("org.apache.hudi.table.format.TestFlinkRowDataReaderContext#configuredDeleteLogPredicates") + void testConfiguredDeleteLogPredicates(FilterPredicate predicate, List<String> expectedKeys) throws Exception { + HoodieLogFile deleteFile = createDeleteFile(); + Configuration conf = storage.getConf().unwrapAs(Configuration.class); + ParquetInputFormat.setFilterPredicate(conf, predicate); + String originalFilter = conf.get(ParquetInputFormat.FILTER_PREDICATE); + // No query predicates: exercise the configured filter and its downstream reload in ParquetFileReader. + assertEquals(expectedKeys, readDeleteKeys(deleteFile, null)); + assertEquals(originalFilter, conf.get(ParquetInputFormat.FILTER_PREDICATE)); + } + + @Test + void testConfiguredFilterRemainsAvailableForDataFiles() throws Exception { + HoodieLogFile deleteFile = createDeleteFile(); + HoodieLogFile dataFile = HoodieFileSliceTestUtils.createNativeDataLogFile( + nativeLogPath(".log.parquet"), Collections.singletonList(record("other", 100L)), HOODIE_SCHEMA, "002"); + Configuration conf = storage.getConf().unwrapAs(Configuration.class); + ParquetInputFormat.setFilterPredicate(conf, keyEquals(ROW_KEY, "k").filter()); + String originalFilter = conf.get(ParquetInputFormat.FILTER_PREDICATE); + assertEquals(Collections.singletonList("k"), readDeleteKeys(deleteFile, null)); + assertEquals(originalFilter, conf.get(ParquetInputFormat.FILTER_PREDICATE)); + FlinkRowDataReaderContext context = readerContext(Collections.emptyList()); + context.setHasLogFiles(true); + try (ClosableIterator<RowData> iterator = context.getFileRecordIterator(dataFile.getPath(), 0, Long.MAX_VALUE, + HOODIE_SCHEMA, HOODIE_SCHEMA, storage)) { + assertFalse(iterator.hasNext(), "The shared configuration must still filter files containing the business key"); + } + } + + private HoodieLogFile createDeleteFile() throws Exception { + return HoodieFileSliceTestUtils.createNativeDeleteLogFile(storage, + nativeLogPath(".deletes.parquet"), Collections.singletonList(record("k", 200L)), HOODIE_SCHEMA, "002"); + } + + private List<String> readDeleteKeys(HoodieLogFile deleteFile, ExpressionPredicates.Predicate predicate) throws Exception { + HoodieSchema schema = HoodieSchemaUtils.createDeleteLogSchema(HOODIE_SCHEMA, Collections.singletonList("timestamp")); + FlinkRowDataReaderContext context = readerContext(predicate == null ? Collections.emptyList() : Collections.singletonList(predicate)); + context.setHasLogFiles(true); + List<String> keys = new ArrayList<>(); + try (ClosableIterator<RowData> iterator = context.getFileRecordIterator(deleteFile.getPath(), 0, Long.MAX_VALUE, + schema, schema, storage)) { + while (iterator.hasNext()) { + keys.add(iterator.next().getString(0).toString()); + } + } + return keys; + } + + private List<String> readSnapshot(HoodieBaseFile baseFile, HoodieLogFile deleteFile, String key) throws Exception { + HoodieFileGroupReader<RowData> reader = HoodieFileGroupReader.<RowData>builder() + .withReaderContext(readerContext(key)) + .withHoodieTableMetaClient(metaClient) + .withStorage(storage) + .withLatestCommitTime("002") + .withBaseFileOption(Option.of(baseFile)) + .withLogFiles(Collections.singletonList(deleteFile).stream()) + .withPartitionPath(PARTITION) + .withDataSchema(HOODIE_SCHEMA) + .withRequestedSchema(HOODIE_SCHEMA) + .withProps(TypedProperties.copy(metaClient.getTableConfig().getProps())) + .withShouldUseRecordPosition(false) + .build(); + List<String> keys = new ArrayList<>(); + try (ClosableIterator<RowData> iterator = reader.getClosableIterator()) { + while (iterator.hasNext()) { + String recordKey = iterator.next().getString(HOODIE_SCHEMA.getField(ROW_KEY).get().pos()).toString(); + // Parquet pushdown prunes row groups; the SQL engine evaluates the predicate after merging. + if (key == null || key.equals(recordKey)) { + keys.add(recordKey); + } + } + } + return keys; + } + + private FlinkRowDataReaderContext readerContext(String key) { + return readerContext(key == null ? Collections.emptyList() : Collections.singletonList(keyEquals(ROW_KEY, key))); + } + + private FlinkRowDataReaderContext readerContext(List<ExpressionPredicates.Predicate> predicates) { + return new FlinkRowDataReaderContext(storage.getConf(), + () -> InternalSchemaManager.DISABLED, predicates, metaClient.getTableConfig(), Option.empty()); + } + + private String nativeLogPath(String extension) { + return new StoragePath(partitionPath, FILE_ID + "_" + DEFAULT_WRITE_TOKEN + "_002_1" + extension).toString(); + } + + private IndexedRecord record(String key, long timestamp) { + return DATA_GEN.generateGenericRecord(key, PARTITION, "rider." + key, "driver." + key, timestamp, false, false); + } + } + + private static Stream<Arguments> configuredDeleteLogPredicates() { + FilterPredicate missingKey = keyEquals(ROW_KEY, "k").filter(); + FilterPredicate storedKey = keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other").filter(); + List<String> retainedKeys = Collections.singletonList("k"); + return Stream.of( + Arguments.of(missingKey, retainedKeys), + Arguments.of(FilterApi.and(missingKey, storedKey), retainedKeys), + Arguments.of(FilterApi.or(missingKey, storedKey), retainedKeys), + Arguments.of(FilterApi.or(storedKey, missingKey), retainedKeys), + Arguments.of(FilterApi.not(FilterApi.and(keyEquals(ROW_KEY, "other").filter(), + keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k").filter())), retainedKeys), + Arguments.of(keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k").filter(), retainedKeys), + Arguments.of(storedKey, Collections.emptyList())); + } + + private static Stream<Arguments> deleteLogPredicates() { + ExpressionPredicates.Predicate missingKey = keyEquals(ROW_KEY, "k"); + ExpressionPredicates.Predicate storedKey = keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "other"); + List<String> retainedKeys = Collections.singletonList("k"); + return Stream.of( + Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates(missingKey, storedKey), retainedKeys), + Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates(storedKey, missingKey), retainedKeys), + Arguments.of(ExpressionPredicates.Not.getInstance().bindPredicate( + ExpressionPredicates.And.getInstance().bindPredicates(keyEquals(ROW_KEY, "other"), + keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "k"))), retainedKeys), + Arguments.of(ExpressionPredicates.Or.getInstance().bindPredicates( + keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "a"), keyEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD, "b")), Collections.emptyList()), + // Physical presence does not make a non-key ordering predicate safe for MOR. + Arguments.of(ExpressionPredicates.Equals.getInstance() + .bindValueLiteral(new ValueLiteralExpression(300L)) + .bindFieldReference(new FieldReferenceExpression("timestamp", DataTypes.BIGINT(), 0, 1)), retainedKeys)); + } + + private static ExpressionPredicates.Predicate keyEquals(String field, String key) { + return ExpressionPredicates.Equals.getInstance() + .bindValueLiteral(new ValueLiteralExpression(key)) + .bindFieldReference(new FieldReferenceExpression(field, DataTypes.STRING(), 0, 0)); + } + private GenericRowData createBaseRow(int id, String name, boolean active) { return GenericRowData.of(id, StringData.fromString(name), active); } diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java index 8086e1516972..a0fa1778dd19 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestRecordIterators.java @@ -39,23 +39,41 @@ import org.apache.hadoop.conf.Configuration; import org.apache.parquet.column.ColumnReader; import org.apache.parquet.filter.RecordFilter; import org.apache.parquet.filter.UnboundRecordFilter; +import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.filter2.predicate.LogicalInverseRewriter; +import org.apache.parquet.filter2.predicate.Operators; +import org.apache.parquet.filter2.predicate.Statistics; +import org.apache.parquet.filter2.predicate.UserDefinedPredicate; +import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter; import org.apache.parquet.hadoop.BadConfigurationException; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.SerializationUtil; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.MockedStatic; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import static org.apache.parquet.filter2.predicate.FilterApi.eq; import static org.apache.parquet.filter2.predicate.FilterApi.intColumn; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; import static org.apache.parquet.hadoop.ParquetInputFormat.FILTER_PREDICATE; import static org.apache.parquet.hadoop.ParquetInputFormat.UNBOUND_RECORD_FILTER; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -63,11 +81,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -124,6 +142,56 @@ class TestRecordIterators { assertTrue(exception.getCause() instanceof RuntimeException); } + @Test + void testMissingPhysicalColumnSkipsPredicateTranslation() { + MessageType schema = MessageTypeParser.parseMessageType("message file { optional int32 stored; }"); + Predicate predicate = mock(Predicate.class); + when(predicate.references()).thenReturn(Collections.singletonList("missing")); + assertNull(RecordIterators.createFilterPredicate(schema, Collections.singletonList(predicate), null)); + verify(predicate, never()).filter(); + } + + @Test + void testPhysicalNestedColumnPathIsSupported() { + MessageType schema = MessageTypeParser.parseMessageType("message file { optional group nested { optional int32 id; } }"); + Predicate predicate = mock(Predicate.class); + FilterPredicate expected = eq(intColumn("nested.id"), 7); + when(predicate.references()).thenReturn(Collections.singletonList("nested.id")); + when(predicate.filter()).thenReturn(expected); + assertSame(expected, RecordIterators.createFilterPredicate(schema, Collections.singletonList(predicate), null)); + + when(predicate.references()).thenReturn(Collections.singletonList("nested.missing")); + assertNull(RecordIterators.createFilterPredicate(schema, Collections.singletonList(predicate), null)); + } + + @ParameterizedTest + @ValueSource(strings = {"stored", "nested.id", "missing", "nested.missing", "nested"}) + void testConfiguredPredicateColumnValidation(String columnName) { + MessageType schema = MessageTypeParser.parseMessageType( + "message file { optional int32 stored; optional group nested { optional int32 id; } }"); + Operators.IntColumn column = intColumn(columnName); + FilterPredicate userDefined = FilterApi.userDefined(column, KeepAllPredicate.class); + List<FilterPredicate> predicates = Arrays.asList( + eq(column, 7), FilterApi.notEq(column, 7), FilterApi.lt(column, 7), FilterApi.ltEq(column, 7), + FilterApi.gt(column, 7), FilterApi.gtEq(column, 7), + FilterApi.in(column, Collections.singleton(7)), FilterApi.notIn(column, Collections.singleton(7)), + FilterApi.and(eq(intColumn("stored"), 1), eq(column, 7)), + FilterApi.or(eq(intColumn("stored"), 1), eq(column, 7)), + FilterApi.not(eq(column, 7)), userDefined, LogicalInverseRewriter.rewrite(FilterApi.not(userDefined))); + boolean expected = columnName.equals("stored") || columnName.equals("nested.id"); + for (FilterPredicate predicate : predicates) { + assertEquals(expected, ParquetPredicateColumnValidator.hasAllColumns(predicate, schema), predicate.toString()); + } + } + + @Test + void testUnknownConfiguredPredicateIsSkipped() { + FilterPredicate predicate = mock(FilterPredicate.class); + when(predicate.accept(any())).thenThrow(new UnsupportedOperationException("Unsupported operator")); + MessageType schema = MessageTypeParser.parseMessageType("message file { optional int32 stored; }"); + assertFalse(ParquetPredicateColumnValidator.hasAllColumns(predicate, schema)); + } + @Test void testUnboundRecordFilterConfiguration() throws Exception { Configuration conf = new Configuration(); @@ -186,15 +254,17 @@ class TestRecordIterators { assertEquals(Collections.singletonMap("region", "us"), partitionSpec); } - @Test - void testParquetIteratorBuildsPredicateAndSchemaEvolutionReader() throws Exception { - InternalSchema mergeSchema = new InternalSchema(Types.RecordType.get( - Types.Field.get(0, false, "id", Types.IntType.get()))); + @ParameterizedTest + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + void testParquetIteratorReusesFooterAndBuildsPredicate(boolean schemaEvolution, boolean configuredMissingColumn) throws Exception { + InternalSchema mergeSchema = schemaEvolution ? new InternalSchema(Types.RecordType.get( + Types.Field.get(0, false, "id", Types.IntType.get()))) : InternalSchema.getEmptyInternalSchema(); InternalSchemaManager schemaManager = mock(InternalSchemaManager.class); CastMap castMap = new CastMap(); DataType[] fieldTypes = {DataTypes.INT()}; castMap.setFileFieldTypes(fieldTypes); Predicate predicate = mock(Predicate.class); + when(predicate.references()).thenReturn(Collections.singletonList("id")); when(predicate.filter()).thenReturn(eq(intColumn("id"), 7)); when(schemaManager.getMergeSchema(anyString())).thenReturn(mergeSchema); when(schemaManager.getCastMap( @@ -208,29 +278,58 @@ class TestRecordIterators { ParquetColumnarRowSplitReader reader = mock(ParquetColumnarRowSplitReader.class); when(reader.reachedEnd()).thenReturn(true); - try (MockedStatic<ParquetSplitReaderUtil> mocked = mockStatic(ParquetSplitReaderUtil.class)) { + Configuration conf = new Configuration(); + if (configuredMissingColumn) { + SerializationUtil.writeObjectToConfAsBase64(FILTER_PREDICATE, eq(intColumn("missing"), 1), conf); + } + String originalFilter = conf.get(FILTER_PREDICATE); + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path("file:///table.parquet"); + MessageType fileSchema = MessageTypeParser.parseMessageType("message file { optional int32 id; }"); + ParquetMetadata footer = new ParquetMetadata(new FileMetaData(fileSchema, Collections.emptyMap(), "test"), Collections.emptyList()); + try (MockedStatic<ParquetFileReader> parquetReader = mockStatic(ParquetFileReader.class); + MockedStatic<ParquetSplitReaderUtil> mocked = mockStatic(ParquetSplitReaderUtil.class)) { + parquetReader.when(() -> ParquetFileReader.readFooter( + org.mockito.ArgumentMatchers.eq(conf), org.mockito.ArgumentMatchers.eq(path), any(MetadataFilter.class))) + .thenAnswer(invocation -> { + assertEquals(range(10L, 30L).toString(), invocation.getArgument(2).toString()); + return footer; + }); mocked.when(() -> ParquetSplitReaderUtil.genPartColumnarRowReader( anyBoolean(), anyBoolean(), any(Configuration.class), any(String[].class), - any(DataType[].class), anyMap(), any(int[].class), anyInt(), any(), anyLong(), - anyLong(), any(), any())).thenReturn(reader); + any(DataType[].class), anyMap(), any(int[].class), anyInt(), any(), any(), + any(), any())).thenAnswer(invocation -> { + assertSame(footer, invocation.getArgument(9)); + assertEquals(eq(intColumn("id"), 7), invocation.getArgument(10)); + Configuration readConf = invocation.getArgument(2); + assertNull(readConf.get(FILTER_PREDICATE)); + if (configuredMissingColumn) { + assertNotSame(conf, readConf); + } else { + assertSame(conf, readConf); + } + return reader; + }); ClosableIterator<RowData> iterator = RecordIterators.getParquetRecordIterator( schemaManager, true, true, - new Configuration(), + conf, new String[] {"id"}, fieldTypes, Collections.emptyMap(), new int[] {0}, 16, - new org.apache.flink.core.fs.Path("file:///table.parquet"), - 0L, - 1L, + new org.apache.flink.core.fs.Path(path.toUri()), + 10L, + 20L, Collections.singletonList(predicate)); assertFalse(iterator.hasNext()); iterator.close(); verify(reader).close(); + assertEquals(originalFilter, conf.get(FILTER_PREDICATE)); + parquetReader.verify(() -> ParquetFileReader.readFooter( + org.mockito.ArgumentMatchers.eq(conf), org.mockito.ArgumentMatchers.eq(path), any(MetadataFilter.class))); } } @@ -244,6 +343,23 @@ class TestRecordIterators { return (T) method.invoke(null, arguments); } + public static class KeepAllPredicate extends UserDefinedPredicate<Integer> { + @Override + public boolean keep(Integer value) { + return true; + } + + @Override + public boolean canDrop(Statistics<Integer> statistics) { + return false; + } + + @Override + public boolean inverseCanDrop(Statistics<Integer> statistics) { + return false; + } + } + public static class ConfigurableRecordFilter implements UnboundRecordFilter, Configurable { private Configuration conf; diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index e702461b5203..f89b2c4ac75a 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index b9f7527a91a8..3454c16e8093 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) { diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index e702461b5203..f89b2c4ac75a 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index b9f7527a91a8..3454c16e8093 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) { diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index e702461b5203..f89b2c4ac75a 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index b9f7527a91a8..3454c16e8093 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) { diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index e702461b5203..f89b2c4ac75a 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -78,6 +78,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -139,8 +140,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -178,8 +178,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index b9f7527a91a8..3454c16e8093 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) { diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index d74f4d71e1ca..998f22dfc9be 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -81,6 +81,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -142,8 +143,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -181,8 +181,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index b9f7527a91a8..3454c16e8093 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) { diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java index d74f4d71e1ca..998f22dfc9be 100644 --- a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java @@ -81,6 +81,7 @@ import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIO; import org.apache.parquet.io.GroupColumnIO; import org.apache.parquet.io.MessageColumnIO; @@ -142,8 +143,7 @@ public class ParquetSplitReaderUtil { int[] selectedFields, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { @@ -181,8 +181,7 @@ public class ParquetSplitReaderUtil { gen, batchSize, new org.apache.hadoop.fs.Path(path.toUri()), - splitStart, - splitLength, + footer, filterPredicate, recordFilter); } diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java index eca0fceae6aa..1436a877195d 100644 --- a/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java +++ b/hudi-flink-datasource/hudi-flink2.2.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java @@ -62,8 +62,6 @@ import java.util.stream.IntStream; import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector; import static org.apache.parquet.filter2.compat.FilterCompat.get; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; /** * This reader is used to read a {@link VectorizedColumnBatch} from input split. @@ -137,14 +135,12 @@ public class ParquetColumnarRowSplitReader implements Closeable { ColumnBatchGenerator generator, int batchSize, Path path, - long splitStart, - long splitLength, + ParquetMetadata footer, FilterPredicate filterPredicate, UnboundRecordFilter recordFilter) throws IOException { this.utcTimestamp = utcTimestamp; this.batchSize = batchSize; // then we need to apply the predicate push down filter - ParquetMetadata footer = readFooter(conf, path, range(splitStart, splitStart + splitLength)); MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = get(filterPredicate, recordFilter); List<BlockMetaData> blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); diff --git a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java index 3763003e554d..8258b2d4cb07 100644 --- a/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java +++ b/hudi-flink-datasource/hudi-flink2.2.x/src/test/java/org/apache/hudi/table/format/cow/TestParquetSplitReaderUtil.java @@ -35,6 +35,7 @@ import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.Test; @@ -51,6 +52,8 @@ import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; +import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -352,6 +355,7 @@ public class TestParquetSplitReaderUtil { } } + ParquetMetadata footer = readFooter(conf, path, range(0, new File(tempDir, "data.parquet").length())); List<String> values = new ArrayList<>(); try (ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader( true, @@ -362,8 +366,7 @@ public class TestParquetSplitReaderUtil { VectorizedColumnBatch::new, batchSize, path, - 0, - new File(tempDir, "data.parquet").length(), + footer, null, null)) { while (!reader.reachedEnd()) {
