github-actions[bot] commented on code in PR #65782:
URL: https://github.com/apache/doris/pull/65782#discussion_r3610329639
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/helper/IcebergWriterHelper.java:
##########
@@ -178,8 +195,95 @@ private static Metrics buildDataFileMetrics(Table table,
FileFormat fileFormat,
}
}
- return new Metrics(commitData.getRowCount(), columnSizes, valueCounts,
- nullValueCounts, null, lowerBounds, upperBounds);
+ // Physical file stats may contain every column, but manifest metrics
must honor the table's metadata policy.
+ return new Metrics(commitData.getRowCount(),
+ filterDisabledMetrics(columnSizes, schema, metricsConfig),
+ filterLogicalMetrics(valueCounts, schema, metricsConfig,
fieldParents),
+ filterLogicalMetrics(nullValueCounts, schema, metricsConfig,
fieldParents),
+ null,
+ filterBounds(lowerBounds, schema, metricsConfig, fieldParents,
true),
+ filterBounds(upperBounds, schema, metricsConfig, fieldParents,
false));
+ }
+
+ private static <T> Map<Integer, T> filterDisabledMetrics(
+ Map<Integer, T> metrics, Schema schema, MetricsConfig
metricsConfig) {
+ Map<Integer, T> filteredMetrics = new HashMap<>();
+ metrics.forEach((fieldId, value) -> {
+ if (MetricsUtil.metricsMode(schema, metricsConfig, fieldId) !=
MetricsModes.None.get()) {
+ filteredMetrics.put(fieldId, value);
+ }
+ });
+ return filteredMetrics;
+ }
+
+ private static <T> Map<Integer, T> filterLogicalMetrics(
+ Map<Integer, T> metrics, Schema schema, MetricsConfig
metricsConfig,
+ Map<Integer, Integer> fieldParents) {
+ Map<Integer, T> filteredMetrics = new HashMap<>();
+ metrics.forEach((fieldId, value) -> {
+ // Definition-level values below list/map do not represent logical
element counts.
+ if (!isInRepeatedField(fieldId, schema, fieldParents)
+ && MetricsUtil.metricsMode(schema, metricsConfig, fieldId)
!= MetricsModes.None.get()) {
+ filteredMetrics.put(fieldId, value);
+ }
+ });
+ return filteredMetrics;
+ }
+
+ private static Map<Integer, ByteBuffer> filterBounds(
+ Map<Integer, ByteBuffer> bounds, Schema schema, MetricsConfig
metricsConfig,
+ Map<Integer, Integer> fieldParents, boolean lowerBound) {
+ Map<Integer, ByteBuffer> filteredBounds = new HashMap<>();
+ bounds.forEach((fieldId, value) -> {
+ if (isInRepeatedField(fieldId, schema, fieldParents)) {
+ return;
+ }
+ MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema,
metricsConfig, fieldId);
+ if (mode == MetricsModes.None.get() || mode ==
MetricsModes.Counts.get()) {
+ return;
+ }
+
+ ByteBuffer filteredValue = value;
+ if (mode instanceof MetricsModes.Truncate) {
+ Type type = schema.findType(fieldId);
+ int length = ((MetricsModes.Truncate) mode).length();
+ // Truncated upper bounds must round up so file pruning cannot
exclude matching values.
+ filteredValue = truncateBound(type, value, length, lowerBound);
+ }
+ if (filteredValue != null) {
+ filteredBounds.put(fieldId, filteredValue);
+ }
+ });
+ return filteredBounds;
+ }
+
+ private static boolean isInRepeatedField(
+ int fieldId, Schema schema, Map<Integer, Integer> fieldParents) {
+ Integer parentId = fieldId;
+ while ((parentId = fieldParents.get(parentId)) != null) {
+ Types.NestedField parent = schema.findField(parentId);
+ if (parent != null && !parent.type().isStructType()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static ByteBuffer truncateBound(Type type, ByteBuffer value, int
length, boolean lowerBound) {
+ switch (type.typeId()) {
+ case STRING:
+ String stringValue = Conversions.fromByteBuffer(type,
value).toString();
Review Comment:
[P1] Do not decode shifted ORC bounds as UTF-8
The ORC collector still indexes top-level fields as `i + 1`, so after a
complex field it can attach a descendant's bytes to the following field. For `s
struct<a:int>, b string` with `s.a=-1`, the second iteration reads `a`'s
integer statistics and stores eight `0xFF` bytes under `b`'s Iceberg ID. Before
this diff those bytes were passed to `Metrics` without decoding; under the
normal `truncate(16)` mode this new call decodes them as UTF-8, and Iceberg's
strict decoder throws `RuntimeIOException`, causing insert/rewrite/merge to
fail during FE commit conversion. Please resolve the actual ORC column ID
before type-aware conversion (or otherwise guarantee the bound type matches the
field), and cover a nested negative integer followed by a string in an ORC
write test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
