amogh-jahagirdar commented on code in PR #18925:
URL: https://github.com/apache/iceberg/pull/18925#discussion_r4574999579


##########
core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java:
##########
@@ -583,7 +597,189 @@ private void wrapInternal(DataFile newFile) {
           newFile.content());
 
       this.file = newFile;
-      this.stats = hasContentStats(newFile) ? statsWrapper.wrap(newFile) : 
null;
+      this.stats = wrapStats(fileWithPartitionSourceStats());
+    }
+
+    private ContentStats wrapStats(DataFile dataFile) {
+      return hasContentStats(dataFile) ? statsWrapper.wrap(dataFile) : null;
+    }
+
+    private DataFile fileWithPartitionSourceStats() {
+      PartitionSpec spec = specsById.get(file.specId());
+      Preconditions.checkArgument(
+          spec != null, "Cannot find partition spec for spec ID: %s", 
file.specId());
+
+      Set<Integer> partitionSourceIds = Sets.newHashSet();
+      Set<Integer> floatingPointSourceIds = Sets.newHashSet();
+      for (PartitionField field : spec.fields()) {
+        if (field.transform().isVoid()) {
+          continue;
+        }
+
+        Preconditions.checkArgument(
+            field.transform().preservesOrder(),
+            "Cannot convert data file %s: partition field does not preserve 
order: %s",
+            file.location(),
+            field);
+        Type sourceType = tableSchema.findType(field.sourceId());
+        Preconditions.checkArgument(
+            sourceType != null,
+            "Cannot convert data file %s: partition source column was dropped: 
%s",
+            file.location(),
+            field);
+
+        partitionSourceIds.add(field.sourceId());
+        if (StatsUtil.isFloatingPoint(sourceType)) {
+          floatingPointSourceIds.add(field.sourceId());
+        }
+      }
+
+      if (partitionSourceIds.isEmpty()) {
+        return file;
+      }
+
+      Map<Integer, Long> valueCounts =
+          fillFromPartition(file.valueCounts(), partitionSourceIds, id -> 
file.recordCount());
+      Map<Integer, Long> nullValueCounts =
+          fillFromPartition(file.nullValueCounts(), partitionSourceIds, 
this::nullValueCount);
+      Map<Integer, Long> nanValueCounts =
+          fillFromPartition(file.nanValueCounts(), floatingPointSourceIds, 
this::nanValueCount);
+      Map<Integer, ByteBuffer> lowerBounds =
+          fillFromPartition(file.lowerBounds(), partitionSourceIds, id -> 
boundsFor(id).lower());
+      Map<Integer, ByteBuffer> upperBounds =
+          fillFromPartition(file.upperBounds(), partitionSourceIds, id -> 
boundsFor(id).upper());
+
+      return new Delegates.DelegatingDataFile(file) {
+        @Override
+        public Map<Integer, Long> valueCounts() {
+          return valueCounts;
+        }
+
+        @Override
+        public Map<Integer, Long> nullValueCounts() {
+          return nullValueCounts;
+        }
+
+        @Override
+        public Map<Integer, Long> nanValueCounts() {
+          return nanValueCounts;
+        }
+
+        @Override
+        public Map<Integer, ByteBuffer> lowerBounds() {
+          return lowerBounds;
+        }
+
+        @Override
+        public Map<Integer, ByteBuffer> upperBounds() {
+          return upperBounds;
+        }
+      };
+    }
+
+    // a null or NaN partition value means every row's source value is null or 
NaN
+    private long nullValueCount(int sourceColumnId) {
+      return partitionValueFor(sourceColumnId) == null ? file.recordCount() : 
0L;
+    }
+
+    private long nanValueCount(int sourceColumnId) {
+      return NaNUtil.isNaN(partitionValueFor(sourceColumnId)) ? 
file.recordCount() : 0L;
+    }
+
+    private StatsUtil.PartitionSourceBounds<ByteBuffer> boundsFor(int 
sourceColumnId) {
+      if (useColumnBounds(sourceColumnId)) {
+        return new StatsUtil.PartitionSourceBounds<>(
+            file.lowerBounds().get(sourceColumnId), 
file.upperBounds().get(sourceColumnId));
+      }
+
+      Type type = tableSchema.findType(sourceColumnId);
+      StatsUtil.PartitionSourceBounds<?> bounds = 
narrowestBoundsFor(sourceColumnId);
+      return new StatsUtil.PartitionSourceBounds<>(
+          Conversions.toByteBuffer(type, bounds.lower()),
+          Conversions.toByteBuffer(type, bounds.upper()));
+    }
+
+    private boolean useColumnBounds(int columnId) {

Review Comment:
   Yeah this is right. For context for string/binary I went for the most 
conservative/simplest approach which is just to use the partition value always 
to derive the stats but you're right that for string/binary we could even 
further have narrower bounds (if the file has a truncate transform on a string 
column with a width lower than what the string/binary may have been truncated 
to via metrics configuration). In any combination, we can always just pick the 
narrowest.
   
   I'd even go further just to generalize the logic to always pick the 
narrowest rather than bifurcate situations where we use the column stats or not 
(regardless of data type). 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to