weijietong commented on code in PR #10836:
URL: https://github.com/apache/paimon/pull/10836#discussion_r4525059191
##########
paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java:
##########
@@ -247,6 +249,56 @@ public FileRecordReader<InternalRow>
createReader(FormatReaderFactory.Context co
ParquetInputFile.fromPath(context.fileIO(),
context.filePath(), context.fileSize());
ParquetReadOptions.Builder readOptionsBuilder =
ParquetUtil.getParquetReadOptionsBuilder(conf).withRange(0,
context.fileSize());
+
+ // Enable selective footer parsing when no predicates are present and
the feature is
+ // enabled.
+ // This improves performance by only parsing metadata for requested
columns.
+ // The effective switch lives in the vendored
ParquetFileReader.readFooter: it checks
+ // SELECTIVE_PARSER_ENABLED_KEY and REQUESTED_COLUMNS_KEY and only
then tries the
+ // selective parse; this factory is just the property producer.
+ // NOTE: When predicates are present, statistics are needed for
row-group filtering,
+ // so selective parser is currently disabled in that case. If
selective parser is
+ // extended to support predicates in the future, set
parseStatistics=true to preserve
+ // statistics metadata.
+ boolean enableSelectiveParser =
+ conf.get(CoreOptions.PARQUET_FOOTER_SELECTIVE_PARSER_ENABLED);
+ // A projected name containing the NUL separator cannot round-trip
through
+ // REQUESTED_COLUMNS_KEY: splitRequestedColumns would split it into
multiple names,
+ // silently rewriting the requested set, and those fragments can match
real columns so
+ // the all-unmatched fallback never runs — the reader then fails on
the missing chunk.
+ // Keep such reads on the official full parse (identical to the option
being disabled).
+ boolean namesEncodable =
+ readType.getFieldNames().stream()
Review Comment:
done
##########
paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java:
##########
@@ -247,6 +249,56 @@ public FileRecordReader<InternalRow>
createReader(FormatReaderFactory.Context co
ParquetInputFile.fromPath(context.fileIO(),
context.filePath(), context.fileSize());
ParquetReadOptions.Builder readOptionsBuilder =
ParquetUtil.getParquetReadOptionsBuilder(conf).withRange(0,
context.fileSize());
+
+ // Enable selective footer parsing when no predicates are present and
the feature is
+ // enabled.
+ // This improves performance by only parsing metadata for requested
columns.
+ // The effective switch lives in the vendored
ParquetFileReader.readFooter: it checks
+ // SELECTIVE_PARSER_ENABLED_KEY and REQUESTED_COLUMNS_KEY and only
then tries the
+ // selective parse; this factory is just the property producer.
+ // NOTE: When predicates are present, statistics are needed for
row-group filtering,
+ // so selective parser is currently disabled in that case. If
selective parser is
+ // extended to support predicates in the future, set
parseStatistics=true to preserve
+ // statistics metadata.
+ boolean enableSelectiveParser =
+ conf.get(CoreOptions.PARQUET_FOOTER_SELECTIVE_PARSER_ENABLED);
+ // A projected name containing the NUL separator cannot round-trip
through
+ // REQUESTED_COLUMNS_KEY: splitRequestedColumns would split it into
multiple names,
+ // silently rewriting the requested set, and those fragments can match
real columns so
+ // the all-unmatched fallback never runs — the reader then fails on
the missing chunk.
+ // Keep such reads on the official full parse (identical to the option
being disabled).
+ boolean namesEncodable =
+ readType.getFieldNames().stream()
+ .noneMatch(
+ name ->
name.contains(SelectiveFooterParser.COLUMN_NAME_SEPARATOR));
+ if (enableSelectiveParser
+ && namesEncodable
+ && (predicates == null || predicates.isEmpty())) {
+
readOptionsBuilder.set(SelectiveFooterParser.SELECTIVE_PARSER_ENABLED_KEY,
"true");
+ StringBuilder requestedColumns = new StringBuilder();
+ for (DataField field : readType.getFields()) {
+ if (requestedColumns.length() > 0) {
+ // Use null character as separator to support column names
containing commas
+
requestedColumns.append(SelectiveFooterParser.COLUMN_NAME_SEPARATOR);
+ }
+ requestedColumns.append(field.name());
+ }
+ readOptionsBuilder.set(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]