weijietong commented on code in PR #10836:
URL: https://github.com/apache/paimon/pull/10836#discussion_r4525061549
##########
paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -229,6 +230,44 @@ private static final ParquetMetadata readFooter(
// Regular file, or encrypted file with plaintext footer
if (!encryptedFooterMode) {
+ // Try selective footer parsing first for better performance.
Opt-in: the enable
+ // flag AND the requested columns must both be present as
read-options properties
+ // (ParquetReaderFactory sets them together when the footer
option is enabled and
+ // no predicates are present). With either one absent, the
official parse below
+ // runs unchanged — the risks of the selective path never
reach the default path.
+ boolean selectiveParserEnabled =
+ "true"
+ .equals(
+ options.getProperty(
+ SelectiveFooterParser
+
.SELECTIVE_PARSER_ENABLED_KEY));
+ String requestedColumns =
+
options.getProperty(SelectiveFooterParser.REQUESTED_COLUMNS_KEY);
+ if (selectiveParserEnabled && requestedColumns != null &&
fileDecryptor == null) {
+ boolean includeStatistics =
+ !"true"
+ .equals(
+ options.getProperty(
+
SelectiveFooterParser.SKIP_STATISTICS_KEY));
+ boolean caseSensitive =
+ !"false"
+ .equals(
+ options.getProperty(
+
SelectiveFooterParser.CASE_SENSITIVE_KEY));
+ try {
+ return SelectiveFooterParser.parse(
Review Comment:
done
##########
paimon-format/src/main/java/org/apache/paimon/format/parquet/metadata/SelectiveFooterParser.java:
##########
@@ -0,0 +1,1015 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.parquet.metadata;
+
+import org.apache.parquet.format.ColumnChunk;
+import org.apache.parquet.format.ColumnMetaData;
+import org.apache.parquet.format.ColumnOrder;
+import org.apache.parquet.format.CompressionCodec;
+import org.apache.parquet.format.ConvertedType;
+import org.apache.parquet.format.DateType;
+import org.apache.parquet.format.DecimalType;
+import org.apache.parquet.format.Encoding;
+import org.apache.parquet.format.EnumType;
+import org.apache.parquet.format.FieldRepetitionType;
+import org.apache.parquet.format.FileMetaData;
+import org.apache.parquet.format.IntType;
+import org.apache.parquet.format.JsonType;
+import org.apache.parquet.format.KeyValue;
+import org.apache.parquet.format.ListType;
+import org.apache.parquet.format.LogicalType;
+import org.apache.parquet.format.MapType;
+import org.apache.parquet.format.NullType;
+import org.apache.parquet.format.PageEncodingStats;
+import org.apache.parquet.format.RowGroup;
+import org.apache.parquet.format.SchemaElement;
+import org.apache.parquet.format.SizeStatistics;
+import org.apache.parquet.format.Statistics;
+import org.apache.parquet.format.StringType;
+import org.apache.parquet.format.TimeType;
+import org.apache.parquet.format.TimeUnit;
+import org.apache.parquet.format.TimestampType;
+import org.apache.parquet.format.Type;
+import org.apache.parquet.format.TypeDefinedOrder;
+import org.apache.parquet.format.UUIDType;
+import org.apache.parquet.format.converter.ParquetMetadataConverter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Selective parser of the parquet thrift footer that materializes only the
requested column chunks'
+ * metadata, optionally skipping their Statistics.
+ *
+ * <p>Instead of materializing the full {@code
org.apache.parquet.format.FileMetaData} tree with the
+ * generated thrift reader (which parses <b>every</b> column chunk's
<b>every</b> field, including
+ * all {@code Statistics}), this parser walks the compact-protocol bytes
itself with {@link
+ * ThriftCompactReader} and:
+ *
+ * <ul>
+ * <li>fully materializes only column chunks whose top-level column is
requested; other column
+ * chunks are still scanned (the compact protocol forces a full byte
pass) but their bytes are
+ * not retained — the transient shell and the pre-{@code path_in_schema}
fields (type,
+ * encodings) are parsed and then discarded, while every post-path field
is skipped;
+ * <li>materializes {@code Statistics}, {@code encoding_stats} and {@code
size_statistics} only
+ * for those kept chunks;
+ * <li>materializes the schema (including the full {@code LogicalType}
union) by hand, so it is
+ * always complete regardless of the column subset.
+ * </ul>
+ *
+ * <p>Object construction (thrift structs → {@code ParquetMetadata}) is
delegated to {@link
+ * ParquetMetadataConverter#fromParquetMetadata(FileMetaData,
InternalFileDecryptor, boolean, Map)}
+ * to reuse the battle-tested conversion, so the result is byte-for-byte
equivalent to the official
+ * reader for the retained subset.
+ *
+ * <p>Encrypted footers and encrypted columns are not supported and signal
{@link
+ * FallbackSignalException} so the caller can retry with the official parser.
+ */
+public final class SelectiveFooterParser {
+
+ /**
+ * ParquetReadOptions property ("true"/"false") switching the selective
parser on at the
+ * consumption site ({@code ParquetFileReader.readFooter}). Mirrors the
key of {@code
+ * CoreOptions#PARQUET_FOOTER_SELECTIVE_PARSER_ENABLED}; the two must stay
in sync. Selective
+ * parsing only runs when this flag and {@link #REQUESTED_COLUMNS_KEY} are
both set — either one
+ * absent keeps the read on the official parse path.
+ */
+ public static final String SELECTIVE_PARSER_ENABLED_KEY =
+ "parquet.footer.selective-parser.enabled";
+
+ /** ParquetReadOptions property carrying the comma-joined requested
top-level column names. */
+ public static final String REQUESTED_COLUMNS_KEY =
"paimon.parquet.footer.requested.columns";
+
+ /**
+ * ParquetReadOptions property ("true"/"false") controlling whether
Statistics of the
+ * <i>requested</i> columns are parsed. The only consumer of footer
statistics is row-group
+ * filtering with a pushdown filter, which never runs together with this
parser — hence AI
+ * training reads (no pushdown) default to skipping them.
+ */
+ public static final String SKIP_STATISTICS_KEY =
"paimon.parquet.footer.skip.statistics";
+
+ /**
+ * ParquetReadOptions property ("true"/"false") controlling whether column
name matching is
+ * case-sensitive. Defaults to "true" for backward compatibility.
+ */
+ public static final String CASE_SENSITIVE_KEY =
"paimon.parquet.footer.case-sensitive";
+
+ /**
+ * Separator character used to join column names in {@link
#REQUESTED_COLUMNS_KEY}. Uses null
+ * character to support column names containing commas.
+ */
+ public static final String COLUMN_NAME_SEPARATOR = "\0";
+
+ /** Raised when the footer needs the official (non-selective) parse path.
*/
+ public static final class FallbackSignalException extends RuntimeException
{
+ FallbackSignalException(String reason) {
+ super(reason);
+ }
+ }
+
+ private final ThriftCompactReader in;
+ private final @Nullable Set<String> requestedColumns;
+ private final boolean includeStatistics;
+ private final boolean caseSensitive;
+ private final ParquetMetadataConverter converter;
+
+ private SelectiveFooterParser(
+ byte[] footer,
+ int offset,
+ int length,
+ @Nullable Set<String> requestedColumns,
+ boolean includeStatistics,
+ boolean caseSensitive,
+ ParquetMetadataConverter converter) {
+ this.in = new ThriftCompactReader(footer, offset, offset + length);
+ this.requestedColumns = requestedColumns;
+ this.includeStatistics = includeStatistics;
+ this.caseSensitive = caseSensitive;
+ this.converter = converter;
+ }
+
+ /**
+ * Parses the footer bytes, materializing Statistics of the requested
columns.
+ *
+ * @param footerBytes raw footer bytes (thrift compact serialized
FileMetaData); array-backed
+ * buffers are read in place without copying
+ * @param requestedColumns top-level column names to keep; {@code null}
parses everything
+ * (equivalent to NO_FILTER, used to validate equivalence)
+ * @param converter the caller's converter, reused for final object
construction
+ */
+ public static ParquetMetadata parse(
+ ByteBuffer footerBytes,
+ @Nullable Set<String> requestedColumns,
+ ParquetMetadataConverter converter)
+ throws IOException {
+ return parse(footerBytes, requestedColumns, true, true, converter);
+ }
+
+ /**
+ * Parses the footer bytes with explicit statistics control.
+ *
+ * @param footerBytes raw footer bytes (thrift compact serialized
FileMetaData); array-backed
+ * buffers are read in place without copying
+ * @param requestedColumns top-level column names to keep; {@code null}
parses everything
+ * (equivalent to NO_FILTER, used to validate equivalence)
+ * @param includeStatistics whether to include statistics in the parsed
metadata
+ * @param converter the caller's converter, reused for final object
construction
+ */
+ public static ParquetMetadata parse(
+ ByteBuffer footerBytes,
+ @Nullable Set<String> requestedColumns,
+ boolean includeStatistics,
+ ParquetMetadataConverter converter)
+ throws IOException {
+ return parse(footerBytes, requestedColumns, includeStatistics, true,
converter);
+ }
+
+ /**
+ * Parses the footer bytes.
+ *
+ * @param includeStatistics whether Statistics of <i>kept</i> columns are
materialized; {@code
+ * false} skips them (AI training reads never consume footer
statistics: the only consumer,
+ * row-group stats filtering, requires a pushdown filter, and
selective parsing is enabled
+ * only when no filter is present)
+ * @param caseSensitive whether column name matching is case-sensitive;
{@code false} uses
+ * case-insensitive matching
+ */
+ public static ParquetMetadata parse(
+ ByteBuffer footerBytes,
+ @Nullable Set<String> requestedColumns,
+ boolean includeStatistics,
+ boolean caseSensitive,
+ ParquetMetadataConverter converter)
+ throws IOException {
+ if (requestedColumns != null && requestedColumns.isEmpty()) {
+ // nothing can be retained: every column chunk would be dropped
and the parse would
+ // end in FallbackSignalException after a full scan — fail fast
instead
+ throw new FallbackSignalException("empty requested columns");
+ }
+ byte[] bytes;
+ int offset;
+ int length = footerBytes.remaining();
+ if (footerBytes.hasArray()) {
+ bytes = footerBytes.array();
+ offset = footerBytes.arrayOffset() + footerBytes.position();
+ } else {
+ bytes = new byte[length];
+ footerBytes.duplicate().get(bytes);
+ offset = 0;
+ }
+ SelectiveFooterParser parser =
+ new SelectiveFooterParser(
+ bytes,
+ offset,
+ length,
+ requestedColumns,
+ includeStatistics,
+ caseSensitive,
+ converter);
+ try {
+ return parser.parseFileMetaData();
+ } catch (FallbackSignalException e) {
+ throw e;
+ } catch (IOException | RuntimeException e) {
+ // any mismatch with the expected structure falls back to the
official parser
+ throw new FallbackSignalException("selective footer parse failed:
" + e);
+ }
+ }
+
+ // ------------------------------------------------------------------
FileMetaData
+
+ private ParquetMetadata parseFileMetaData() throws IOException {
+ FileMetaData fileMetaData = new FileMetaData();
+ List<RowGroup> rowGroups = new ArrayList<>();
+ Map<RowGroup, Long> rowGroupOffsets = new HashMap<>();
+
+ in.structBegin();
+ int header;
+ while ((header = in.readFieldHeader()) != ThriftCompactReader.STOP) {
+ int id = ThriftCompactReader.fieldId(header);
+ byte type = ThriftCompactReader.type(header);
+ switch (id) {
+ case 1: // version
+ fileMetaData.setVersion(in.readI32());
+ break;
+ case 2: // schema
+ fileMetaData.setSchema(readSchemaElements());
+ break;
+ case 3: // num_rows
+ fileMetaData.setNum_rows(in.readI64());
+ break;
+ case 4: // row_groups
+ readRowGroups(rowGroups, rowGroupOffsets);
+ break;
+ case 5: // key_value_metadata
+ fileMetaData.setKey_value_metadata(readKeyValues());
+ break;
+ case 6: // created_by
+ fileMetaData.setCreated_by(in.readString());
+ break;
+ case 7: // column_orders
+ fileMetaData.setColumn_orders(readColumnOrders());
+ break;
+ case 8: // encryption_algorithm (plaintext footer with
encrypted columns)
+ throw new FallbackSignalException("footer carries
encryption_algorithm");
+ default: // 9: footer_signing_key_metadata and unknown future
fields
+ in.skip(type);
+ }
+ }
+ in.structEnd();
+ fileMetaData.setRow_groups(rowGroups);
+ return converter.fromParquetMetadata(fileMetaData, null, false,
rowGroupOffsets);
+ }
+
+ // ------------------------------------------------------------------
schema (manual)
+
+ private List<SchemaElement> readSchemaElements() throws IOException {
+ int listHeader = in.readListHeader();
+ if (ThriftCompactReader.listElemType(listHeader) !=
ThriftCompactReader.STRUCT) {
+ throw new FallbackSignalException("schema element list of
non-struct type");
+ }
+ int size = ThriftCompactReader.listSize(listHeader);
+ List<SchemaElement> result = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ result.add(readSchemaElement());
+ }
+ return result;
+ }
+
+ private SchemaElement readSchemaElement() throws IOException {
+ SchemaElement element = new SchemaElement();
+ in.structBegin();
+ int header;
+ while ((header = in.readFieldHeader()) != ThriftCompactReader.STOP) {
+ int id = ThriftCompactReader.fieldId(header);
+ byte type = ThriftCompactReader.type(header);
+ switch (id) {
+ case 1: // type (enum)
+ element.setType(Type.findByValue(in.readI32()));
+ break;
+ case 2: // type_length
+ element.setType_length(in.readI32());
+ break;
+ case 3: // repetition_type (enum)
+
element.setRepetition_type(FieldRepetitionType.findByValue(in.readI32()));
+ break;
+ case 4: // name
+ element.setName(in.readString());
+ break;
+ case 5: // num_children
+ element.setNum_children(in.readI32());
+ break;
+ case 6: // converted_type (enum)
+
element.setConverted_type(ConvertedType.findByValue(in.readI32()));
+ break;
+ case 7: // scale
+ element.setScale(in.readI32());
+ break;
+ case 8: // precision
+ element.setPrecision(in.readI32());
+ break;
+ case 9: // field_id
+ element.setField_id(in.readI32());
+ break;
+ case 10: // logicalType (union)
+ element.setLogicalType(readLogicalType());
+ break;
+ default:
+ // unknown future fields are skipped, matching the
official generated reader
+ in.skip(type);
+ }
+ }
+ in.structEnd();
+ return element;
+ }
+
+ /**
+ * Reads a LogicalType union. Exactly one member is set; members are
mostly empty structs, so
+ * each case just consumes the inner struct. Unknown members are skipped
(the official generated
+ * reader would skip them too), keeping the two parsers behavior-identical.
+ */
+ private LogicalType readLogicalType() throws IOException {
+ LogicalType logicalType = new LogicalType();
+ in.structBegin();
+ int header;
+ while ((header = in.readFieldHeader()) != ThriftCompactReader.STOP) {
+ int id = ThriftCompactReader.fieldId(header);
+ switch (id) {
+ case 1: // STRING
+ skipEmptyStruct();
+ logicalType.setSTRING(new StringType());
+ break;
+ case 2: // MAP
+ skipEmptyStruct();
+ logicalType.setMAP(new MapType());
+ break;
+ case 3: // LIST
+ skipEmptyStruct();
+ logicalType.setLIST(new ListType());
+ break;
+ case 4: // ENUM
+ skipEmptyStruct();
+ logicalType.setENUM(new EnumType());
+ break;
+ case 5: // DECIMAL {1 scale, 2 precision}
+ {
+ DecimalType decimal = new DecimalType();
+ in.structBegin();
+ int inner;
+ while ((inner = in.readFieldHeader()) !=
ThriftCompactReader.STOP) {
+ switch (ThriftCompactReader.fieldId(inner)) {
+ case 1:
+ decimal.setScale(in.readI32());
+ break;
+ case 2:
+ decimal.setPrecision(in.readI32());
+ break;
+ default:
+ in.skip(ThriftCompactReader.type(inner));
+ }
+ }
+ in.structEnd();
+ logicalType.setDECIMAL(decimal);
+ break;
+ }
+ case 6: // DATE
+ skipEmptyStruct();
+ logicalType.setDATE(new DateType());
+ break;
+ case 7: // TIME {1 isAdjustedToUTC (bool header), 2 unit
TimeUnit}
+ {
+ TimeType time = new TimeType();
+ readTimeUnitStruct(time, null);
+ logicalType.setTIME(time);
+ break;
+ }
+ case 8: // TIMESTAMP {1 isAdjustedToUTC, 2 unit}
+ {
+ TimestampType timestamp = new TimestampType();
+ readTimeUnitStruct(null, timestamp);
+ logicalType.setTIMESTAMP(timestamp);
+ break;
+ }
+ case 10: // INTEGER {1 bitWidth (byte), 2 isSigned (bool)}
+ {
+ IntType intType = new IntType();
+ in.structBegin();
+ int inner;
+ while ((inner = in.readFieldHeader()) !=
ThriftCompactReader.STOP) {
+ byte innerType = ThriftCompactReader.type(inner);
+ switch (ThriftCompactReader.fieldId(inner)) {
+ case 1:
+ // i8 values are a single raw byte in the
compact protocol
+ intType.setBitWidth(in.readByteValue());
+ break;
+ case 2:
+ intType.setIsSigned(
+ innerType ==
ThriftCompactReader.BOOLEAN_TRUE);
+ break;
+ default:
+ in.skip(innerType);
+ }
+ }
+ in.structEnd();
+ logicalType.setINTEGER(intType);
+ break;
+ }
+ case 11: // UNKNOWN
+ skipEmptyStruct();
+ logicalType.setUNKNOWN(new NullType());
+ break;
+ case 12: // JSON
+ skipEmptyStruct();
+ logicalType.setJSON(new JsonType());
+ break;
+ case 13: // BSON
+ skipEmptyStruct();
+ logicalType.setBSON(new
org.apache.parquet.format.BsonType());
+ break;
+ case 14: // UUID
+ skipEmptyStruct();
+ logicalType.setUUID(new UUIDType());
+ break;
+ case 15: // FLOAT16
+ skipEmptyStruct();
+ logicalType.setFLOAT16(new
org.apache.parquet.format.Float16Type());
+ break;
+ default:
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]