deniskuzZ commented on code in PR #6662:
URL: https://github.com/apache/hive/pull/6662#discussion_r3710632859
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java:
##########
@@ -596,37 +637,78 @@ public Map<String, String> computeBasicStatistics(Partish
partish) {
.commit();
}
}
- return getBasicStatistics(partish);
+ return getBasicStatistics(hmsTable);
}
- private static Map<String, String> getPartishSummary(Partish partish, Table
table, Snapshot snapshot) {
- if (partish.getPartition() != null) {
- Types.StructType partitionType = Partitioning.partitionType(table);
- try (CloseableIterable<PartitionStatistics> records =
-
table.newPartitionStatisticsScan().useSnapshot(snapshot.snapshotId()).scan()) {
- Iterator<PartitionStatistics> recordsIterator = records.iterator();
- if (!recordsIterator.hasNext()) {
- LOG.warn("Partition stats file not found for snapshot: {}",
snapshot.snapshotId());
- return null;
- }
- String partName = partish.getPartition().getName();
- while (recordsIterator.hasNext()) {
- PartitionStatistics stats = recordsIterator.next();
- PartitionSpec spec = table.specs().get(stats.specId());
- PartitionData data =
IcebergTableUtil.toPartitionData(stats.partition(), partitionType,
- spec.partitionType());
- if (spec.partitionToPath(data).equals(partName)) {
- return IcebergTableUtil.toStatsMap(stats);
- }
- }
+ @Override
+ public Map<String, Map<String, String>>
getAggrBasicStatsFor(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
+ List<String> partNames) {
+ if (!HiveMetaHook.ICEBERG.equals(getStatsSource())) {
+ return Map.of();
+ }
+ Table table = getTable(hmsTable);
+ Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable);
- LOG.warn("Partition {} not found in partition stats, falling back to
metadata scan", partName);
- return IcebergTableUtil.getPartitionStats(table,
partish.getPartition().getSpec(), snapshot);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
+ Map<String, Map<String, String>> result =
Maps.newHashMapWithExpectedSize(partNames.size());
+ if (snapshot == null) {
+ partNames.forEach(partName -> result.put(partName, emptyStatsMap()));
+ return result;
+ }
+ collectPartitionStatsFor(table, snapshot, partNames,
+ (partName, stats) -> result.put(partName, toStatsMap(stats)));
+ return result;
+ }
+
+ private PartitionStatistics
getPartitionStatsFor(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
+ String partName) {
+ if (!HiveMetaHook.ICEBERG.equals(getStatsSource())) {
+ return null;
+ }
+ Table table = getTable(hmsTable);
+ Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable);
+ return snapshot != null ?
+ getOrCachePartitionStats(table, snapshot).get(partName) : null;
+ }
+
+ /**
+ * Passes each named partition's statistics to {@code statsConsumer}, served
from the snapshot's partition
+ * statistics file. Partitions missing from the file are skipped and logged.
+ */
+ private void collectPartitionStatsFor(Table table, Snapshot snapshot,
List<String> partNames,
+ BiConsumer<String, PartitionStatistics> statsConsumer) {
+ Map<String, PartitionStatistics> partitionStats =
getOrCachePartitionStats(table, snapshot);
+ int missing = 0;
+
+ for (String partName : partNames) {
+ PartitionStatistics stats = partitionStats.get(partName);
+ if (stats != null) {
+ statsConsumer.accept(partName, stats);
+ } else {
+ missing++;
}
}
- return snapshot.summary();
+ if (missing > 0) {
+ LOG.warn("{} of {} partitions not found in the partition stats file of
snapshot {}",
+ missing, partNames.size(), snapshot.snapshotId());
+ }
Review Comment:
Deliberate.
The fallback fired only when a partition the pruner saw is absent from the
snapshot's stats file. The stats file and the PARTITIONS metadata table are
derived from the same manifests, so that condition means the stats file is
corrupt/incomplete.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
