github-actions[bot] commented on code in PR #65782:
URL: https://github.com/apache/doris/pull/65782#discussion_r3610632183
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java:
##########
@@ -178,8 +195,95 @@ private static Metrics buildDataFileMetrics(Table table,
FileFormat fileFormat,
}
}
- return new Metrics(commitData.getRowCount(), columnSizes, valueCounts,
- nullValueCounts, null, lowerBounds, upperBounds);
+ // Physical file stats may contain every column, but manifest metrics
must honor the table's metadata policy.
+ return new Metrics(commitData.getRowCount(),
+ filterDisabledMetrics(columnSizes, schema, metricsConfig),
+ filterLogicalMetrics(valueCounts, schema, metricsConfig,
fieldParents),
+ filterLogicalMetrics(nullValueCounts, schema, metricsConfig,
fieldParents),
+ null,
+ filterBounds(lowerBounds, schema, metricsConfig, fieldParents,
true),
+ filterBounds(upperBounds, schema, metricsConfig, fieldParents,
false));
+ }
+
+ private static <T> Map<Integer, T> filterDisabledMetrics(
+ Map<Integer, T> metrics, Schema schema, MetricsConfig
metricsConfig) {
+ Map<Integer, T> filteredMetrics = new HashMap<>();
+ metrics.forEach((fieldId, value) -> {
+ if (MetricsUtil.metricsMode(schema, metricsConfig, fieldId) !=
MetricsModes.None.get()) {
+ filteredMetrics.put(fieldId, value);
+ }
+ });
+ return filteredMetrics;
+ }
+
+ private static <T> Map<Integer, T> filterLogicalMetrics(
+ Map<Integer, T> metrics, Schema schema, MetricsConfig
metricsConfig,
+ Map<Integer, Integer> fieldParents) {
+ Map<Integer, T> filteredMetrics = new HashMap<>();
+ metrics.forEach((fieldId, value) -> {
+ // Definition-level values below list/map do not represent logical
element counts.
+ if (!isInRepeatedField(fieldId, schema, fieldParents)
+ && MetricsUtil.metricsMode(schema, metricsConfig, fieldId)
!= MetricsModes.None.get()) {
+ filteredMetrics.put(fieldId, value);
+ }
+ });
+ return filteredMetrics;
+ }
+
+ private static Map<Integer, ByteBuffer> filterBounds(
+ Map<Integer, ByteBuffer> bounds, Schema schema, MetricsConfig
metricsConfig,
+ Map<Integer, Integer> fieldParents, boolean lowerBound) {
+ Map<Integer, ByteBuffer> filteredBounds = new HashMap<>();
+ bounds.forEach((fieldId, value) -> {
+ if (isInRepeatedField(fieldId, schema, fieldParents)) {
+ return;
+ }
+ MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema,
metricsConfig, fieldId);
+ if (mode == MetricsModes.None.get() || mode ==
MetricsModes.Counts.get()) {
+ return;
+ }
+
+ ByteBuffer filteredValue = value;
+ if (mode instanceof MetricsModes.Truncate) {
+ Type type = schema.findType(fieldId);
+ int length = ((MetricsModes.Truncate) mode).length();
+ // Truncated upper bounds must round up so file pruning cannot
exclude matching values.
+ filteredValue = truncateBound(type, value, length, lowerBound);
+ }
+ if (filteredValue != null) {
+ filteredBounds.put(fieldId, filteredValue);
+ }
+ });
+ return filteredBounds;
+ }
+
+ private static boolean isInRepeatedField(
+ int fieldId, Schema schema, Map<Integer, Integer> fieldParents) {
+ Integer parentId = fieldId;
+ while ((parentId = fieldParents.get(parentId)) != null) {
+ Types.NestedField parent = schema.findField(parentId);
+ if (parent != null && !parent.type().isStructType()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static ByteBuffer truncateBound(Type type, ByteBuffer value, int
length, boolean lowerBound) {
+ switch (type.typeId()) {
+ case STRING:
+ String stringValue = Conversions.fromByteBuffer(type,
value).toString();
+ String truncatedString = lowerBound
+ ? UnicodeUtil.truncateStringMin(stringValue, length)
+ : UnicodeUtil.truncateStringMax(stringValue, length);
Review Comment:
[P2] Preserve ORC's no-successor upper-bound fallback
This shared helper now applies the Parquet null result to ORC as well. For
an ORC string maximum whose first retained code point is `U+10FFFF` (for
example `U+10FFFF` followed by another character with `truncate(1)`),
`UnicodeUtil.truncateStringMax` cannot increment the prefix and returns null,
so `filterBounds` drops the upper bound. Iceberg 1.10.1's ORC path deliberately
falls back to the original full maximum in this case, unlike its Parquet path;
otherwise ORC manifests lose the requested upper metric and
pruning/readable-metrics information. Please pass the file format into the
bound filtering and retain the original ORC upper string value when no safe
truncated successor exists, with an ORC-backed edge test. This is distinct from
the prior thread about not truncating bounds at all: it is the ORC-only
fallback lost by the new unified fix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
