difin commented on code in PR #6716:
URL: https://github.com/apache/hive/pull/6716#discussion_r4040956886
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java:
##########
@@ -133,24 +138,90 @@ private static StatisticsFile colStatsFileOf(Table table,
long snapshotId, boole
/**
* The file whose blobs are Hive's own - Iceberg keeps statistics of its own
in the same format -
- * at the asked-for granularity: a blob describing one partition names it in
its metadata.
+ * at the asked-for granularity: a blob describing one partition is of a
type of its own.
*
- * <p>A file that holds any partition is a per partition one, whatever else
it holds. The entries
- * it aggregates from them state the table only while it holds every
partition, which a gather of some
- * of them does not, so a whole-table read passes it by and takes the file
gathered as one.
+ * <p>A file that holds any partition is a per partition one, whatever else
it holds. Its
+ * aggregates serve a whole-table read only while they aggregate the full
table - what a gather
+ * over every partition marked on it, and a gather of some of them did not.
*/
private static boolean holdsHiveColStats(StatisticsFile stats, boolean
partitionLevel) {
boolean holdsPartitions = stats.blobMetadata().stream()
- .anyMatch(metadata ->
metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_FIELD));
+ .anyMatch(metadata ->
IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type()));
if (partitionLevel) {
- return holdsPartitions && stats.blobMetadata().stream().anyMatch(
- metadata ->
IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type()));
+ return holdsPartitions;
}
- return !holdsPartitions && stats.blobMetadata().stream().anyMatch(
+ if (holdsPartitions) {
+ return hasFullTableAggr(stats);
+ }
+ return stats.blobMetadata().stream().anyMatch(
metadata ->
IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) ||
IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(metadata.type()));
}
+ /** Whether the file's aggregates answer for the whole table: its registered
entry says so. */
+ static boolean hasFullTableAggr(StatisticsFile stats) {
+ return stats != null && stats.blobMetadata().stream().anyMatch(
+ metadata ->
"true".equals(metadata.properties().get(IcebergColStatsWriter.FULL_TABLE_AGGR_PROP)));
+ }
+
+ /**
+ * The stored table-level file, taken as it was gathered: a write merges
only into a file
+ * gathered as one, where a read may also take a partition-level file's
aggregates.
+ */
+ static StatisticsFile getTableOnlyColStatsFile(Table table, long snapshotId)
{
+ StatisticsFile stats = getColStatsFile(table, snapshotId, false);
+ return stats == null || stats.blobMetadata().stream()
+ .anyMatch(metadata ->
IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())) ?
+ null : stats;
+ }
+
+ /**
+ * Whether the stored column statistics answer for the column: they still
describe the snapshot
+ * the table names, and their file holds an entry for it. The footer names
the measured columns -
+ * a table-level file on each blob, a partition-level file on its first, so
a column the schema
+ * gained since the write, which moved no snapshot, is refused either way.
+ */
+ public static boolean
colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table hmsTable,
+ List<String> colNames, Configuration conf) {
+ Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
+ Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable);
+ if (snapshot == null) {
+ return false;
+ }
+ Set<Integer> stored = storedFieldIds(table, snapshot, conf);
+ return colNames.stream().allMatch(colName -> {
+ Types.NestedField field =
table.schema().caseInsensitiveFindField(colName);
+ return field != null && stored.contains(field.fieldId());
+ });
+ }
+
+ /**
+ * The fields the stored statistics state for the snapshot. Which file
answers is the same
Review Comment:
Which file answers ... - The wording a little awkward
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
