huaxingao commented on code in PR #18885:
URL: https://github.com/apache/iceberg/pull/18885#discussion_r4546483961
##########
core/src/main/java/org/apache/iceberg/SchemaUpdate.java:
##########
@@ -506,22 +506,33 @@ private TableMetadata
applyChangesToMetadata(TableMetadata metadata) {
// Transform the metrics if they exist
if (base != null && base.properties() != null) {
- Schema newSchema = newMetadata.schema();
- List<String> deletedColumns =
-
deletes.stream().map(schema::findColumnName).collect(Collectors.toList());
- Map<String, String> renamedColumns =
- updates.keySet().stream()
- .filter(id -> !addedNameToId.containsValue(id)) // remove added
columns
- .filter(id ->
!schema.findColumnName(id).equals(newSchema.findColumnName(id)))
- .collect(Collectors.toMap(schema::findColumnName,
newSchema::findColumnName));
+ Map<Integer, String> newIdToName = newMetadata.schema().idToName();
Review Comment:
Nit: the comment above, "Transform the metrics if they exist", now only
covers part of what this block does. Something like "Keep name-keyed per-column
properties in sync" would fit better.
##########
core/src/main/java/org/apache/iceberg/SchemaUpdate.java:
##########
@@ -506,22 +506,33 @@ private TableMetadata
applyChangesToMetadata(TableMetadata metadata) {
// Transform the metrics if they exist
if (base != null && base.properties() != null) {
- Schema newSchema = newMetadata.schema();
- List<String> deletedColumns =
-
deletes.stream().map(schema::findColumnName).collect(Collectors.toList());
- Map<String, String> renamedColumns =
- updates.keySet().stream()
- .filter(id -> !addedNameToId.containsValue(id)) // remove added
columns
- .filter(id ->
!schema.findColumnName(id).equals(newSchema.findColumnName(id)))
- .collect(Collectors.toMap(schema::findColumnName,
newSchema::findColumnName));
+ Map<Integer, String> newIdToName = newMetadata.schema().idToName();
+ List<String> deletedColumns = Lists.newArrayList();
+ Map<String, String> renamedColumns = Maps.newHashMap();
+ // compare all fields so that nested fields of a deleted or renamed
struct are included
+ schema
+ .idToName()
+ .forEach(
+ (id, name) -> {
+ String newName = newIdToName.get(id);
+ if (newName == null) {
+ deletedColumns.add(name);
+ } else if (!newName.equals(name)) {
+ renamedColumns.put(name, newName);
+ }
+ });
+
if (!deletedColumns.isEmpty() || !renamedColumns.isEmpty()) {
Set<String> columnProperties =
ImmutableSet.of(
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX,
TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX,
+ TableProperties.PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX,
Review Comment:
Nit: could we move this set to a constant at the top of the class? This PR
adds three prefixes that were missing from this set. A named constant would
make the set easier to find, so new prefixes are less likely to be missed. For
example:
```java
private static final Set<String> COLUMN_PROPERTY_PREFIXES =
ImmutableSet.of(
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX,
TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX,
TableProperties.PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX,
TableProperties.PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX,
TableProperties.PARQUET_COLUMN_STATS_ENABLED_PREFIX,
TableProperties.PARQUET_COLUMN_COMPRESSION_CODEC_PREFIX,
TableProperties.PARQUET_COLUMN_COMPRESSION_LEVEL_PREFIX,
TableProperties.PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX);
```
and then use it in the `applySchemaChanges` call:
```java
PropertyUtil.applySchemaChanges(
newMetadata.properties(), deletedColumns, renamedColumns,
COLUMN_PROPERTY_PREFIXES);
```
##########
core/src/test/java/org/apache/iceberg/TestSchemaAndMappingUpdate.java:
##########
@@ -229,6 +236,84 @@ public void testModificationWithParquetColumnCompression()
{
.doesNotContainKey(PARQUET_COLUMN_COMPRESSION_LEVEL_PREFIX + "ID");
}
+ @TestTemplate
+ public void testModificationWithParquetBloomFilterFppAndNdv() {
+ table
+ .updateProperties()
+ .set(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "id", "0.05")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "id", "1000")
+ .commit();
+
+ table.updateSchema().renameColumn("id", "ID").commit();
+ assertThat(table.properties())
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "ID", "0.05")
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "ID", "1000")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "id")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "id");
+
+ table.updateSchema().deleteColumn("ID").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "ID")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "ID");
+ }
+
+ @TestTemplate
+ public void testModificationWithParquetDictEncoding() {
+ table
+ .updateProperties()
+ .set(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "id", "false")
+ .commit();
+
+ table.updateSchema().renameColumn("id", "ID").commit();
+ assertThat(table.properties())
+ .containsEntry(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "ID",
"false")
+ .doesNotContainKey(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "id");
+
+ table.updateSchema().deleteColumn("ID").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "ID");
+ }
+
+ @TestTemplate
+ public void testRenameStructWithNestedColumnProperties() {
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ table
+ .updateProperties()
+ .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "location.lat",
"full")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "location.lat",
"true")
+ .commit();
+
+ table.updateSchema().renameColumn("location", "loc").commit();
+ assertThat(table.properties())
+ .containsEntry(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"loc.lat", "full")
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "loc.lat",
"true")
+ .doesNotContainKey(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"location.lat")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
+
+ // a stale metrics property would fail validation of later property updates
+ table.updateProperties().set("key", "value").commit();
+ assertThat(table.properties()).containsEntry("key", "value");
+ }
+
+ @TestTemplate
+ public void testDeleteStructWithNestedColumnProperties() {
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ table
+ .updateProperties()
+ .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "location.lat",
"full")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "location.lat",
"true")
+ .commit();
+
+ table.updateSchema().deleteColumn("location").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"location.lat")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
+
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
Review Comment:
Nit: could this also check the metrics key after re-adding `location`, like
the rename test does?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
