This is an automated email from the ASF dual-hosted git repository. collado pushed a commit to branch mcollado-table-metadata-properties in repository https://gitbox.apache.org/repos/asf/polaris.git
commit e36e7ebc8aac5e92286f394f62968c6e73058a02 Author: Michael Collado <[email protected]> AuthorDate: Thu Oct 2 11:29:16 2025 -0700 Made table properties constants and pulled out static utility method --- .../core/entity/table/IcebergTableLikeEntity.java | 41 ++++++++++++-- .../core/persistence/cache/EntityWeigherTest.java | 2 +- .../service/catalog/iceberg/IcebergCatalog.java | 62 +++++++++++++--------- .../iceberg/AbstractIcebergCatalogTest.java | 25 +++++---- 4 files changed, 90 insertions(+), 40 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java index ca6841b2c..f1b756779 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java @@ -22,6 +22,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Preconditions; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; @@ -32,10 +35,6 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - /** * An entity type for {@link TableLikeEntity} instances that conform to iceberg semantics around * locations. This includes both Iceberg tables and Iceberg views. @@ -51,6 +50,40 @@ public class IcebergTableLikeEntity extends TableLikeEntity { public static final String LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY = "last-notification-timestamp"; + /* + * The following constants are copied from the TableMetadataParser in Iceberg + * They represent the keys used in the table metadata JSON file. + */ + + public static final String FORMAT_VERSION = "format-version"; + public static final String TABLE_UUID = "table-uuid"; + public static final String LOCATION = "location"; + public static final String LAST_SEQUENCE_NUMBER = "last-sequence-number"; + public static final String LAST_UPDATED_MILLIS = "last-updated-ms"; + public static final String LAST_COLUMN_ID = "last-column-id"; + public static final String SCHEMA = "schema"; + public static final String SCHEMAS = "schemas"; + public static final String CURRENT_SCHEMA_ID = "current-schema-id"; + public static final String PARTITION_SPEC = "partition-spec"; + public static final String PARTITION_SPECS = "partition-specs"; + public static final String DEFAULT_SPEC_ID = "default-spec-id"; + public static final String LAST_PARTITION_ID = "last-partition-id"; + public static final String DEFAULT_SORT_ORDER_ID = "default-sort-order-id"; + public static final String SORT_ORDERS = "sort-orders"; + public static final String PROPERTIES = "properties"; + public static final String CURRENT_SNAPSHOT_ID = "current-snapshot-id"; + public static final String REFS = "refs"; + public static final String SNAPSHOTS = "snapshots"; + public static final String SNAPSHOT_ID = "snapshot-id"; + public static final String TIMESTAMP_MS = "timestamp-ms"; + public static final String SNAPSHOT_LOG = "snapshot-log"; + public static final String METADATA_FILE = "metadata-file"; + public static final String METADATA_LOG = "metadata-log"; + public static final String STATISTICS = "statistics"; + public static final String PARTITION_STATISTICS = "partition-statistics"; + public static final String ENCRYPTION_KEYS = "encryption-keys"; + public static final String NEXT_ROW_ID = "next-row-id"; + public IcebergTableLikeEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); PolarisEntitySubType subType = getSubType(); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java index fe965385b..780fbc944 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/EntityWeigherTest.java @@ -124,7 +124,7 @@ public class EntityWeigherTest { "location", "{\"a\": \"b\"}", Optional.of("{\"c\": \"d\", \"e\": \"f\"}"))); - Assertions.assertThat(preciseWeight).isEqualTo(1255); // :( this is hard-coded + Assertions.assertThat(preciseWeight).isEqualTo(1183); // :( this is hard-coded } private static Map<String, String> getPropertiesMap(String properties) { diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 0d1d6979c..f02c61715 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -1566,32 +1566,7 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog "Generic table with same name already exists: %s", tableIdentifier); } } - Map<String, String> storedProperties = new HashMap<>(); - storedProperties.put("location", metadata.location()); - storedProperties.put("format-version", String.valueOf(metadata.formatVersion())); - storedProperties.put("table-uuid", metadata.uuid()); - storedProperties.put("current-schema-id", String.valueOf(metadata.currentSchemaId())); - if (metadata.currentSnapshot() != null) { - storedProperties.put( - "current-snapshot-id", String.valueOf(metadata.currentSnapshot().snapshotId())); - } - storedProperties.put("last-column-id", String.valueOf(metadata.lastColumnId())); - storedProperties.put("next-row-id", String.valueOf(metadata.nextRowId())); - storedProperties.put("last-sequence-number", String.valueOf(metadata.lastSequenceNumber())); - storedProperties.put("last-updated-ms", String.valueOf(metadata.lastUpdatedMillis())); - if (metadata.sortOrder() != null) { - storedProperties.put( - "default-sort-order-id", String.valueOf(metadata.defaultSortOrderId())); - } - if (metadata.spec() != null) { - storedProperties.put("default-spec-id", String.valueOf(metadata.defaultSpecId())); - storedProperties.put( - "last-partition-id", String.valueOf(metadata.lastAssignedPartitionId())); - } - if (metadata.currentSnapshot() != null) { - storedProperties.put( - "current-snapshot-id", String.valueOf(metadata.currentSnapshot().snapshotId())); - } + Map<String, String> storedProperties = buildTableMetadataPropertiesMap(metadata); IcebergTableLikeEntity entity = IcebergTableLikeEntity.of(resolvedPath == null ? null : resolvedPath.getRawLeafEntity()); String existingLocation; @@ -1743,6 +1718,41 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog } } + private static Map<String, String> buildTableMetadataPropertiesMap(TableMetadata metadata) { + Map<String, String> storedProperties = new HashMap<>(); + storedProperties.put(IcebergTableLikeEntity.LOCATION, metadata.location()); + storedProperties.put( + IcebergTableLikeEntity.FORMAT_VERSION, String.valueOf(metadata.formatVersion())); + storedProperties.put(IcebergTableLikeEntity.TABLE_UUID, metadata.uuid()); + storedProperties.put( + IcebergTableLikeEntity.CURRENT_SCHEMA_ID, String.valueOf(metadata.currentSchemaId())); + if (metadata.currentSnapshot() != null) { + storedProperties.put( + IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID, + String.valueOf(metadata.currentSnapshot().snapshotId())); + } + storedProperties.put( + IcebergTableLikeEntity.LAST_COLUMN_ID, String.valueOf(metadata.lastColumnId())); + storedProperties.put(IcebergTableLikeEntity.NEXT_ROW_ID, String.valueOf(metadata.nextRowId())); + storedProperties.put( + IcebergTableLikeEntity.LAST_SEQUENCE_NUMBER, String.valueOf(metadata.lastSequenceNumber())); + storedProperties.put( + IcebergTableLikeEntity.LAST_UPDATED_MILLIS, String.valueOf(metadata.lastUpdatedMillis())); + if (metadata.sortOrder() != null) { + storedProperties.put( + IcebergTableLikeEntity.DEFAULT_SORT_ORDER_ID, + String.valueOf(metadata.defaultSortOrderId())); + } + if (metadata.spec() != null) { + storedProperties.put( + IcebergTableLikeEntity.DEFAULT_SPEC_ID, String.valueOf(metadata.defaultSpecId())); + storedProperties.put( + IcebergTableLikeEntity.LAST_PARTITION_ID, + String.valueOf(metadata.lastAssignedPartitionId())); + } + return storedProperties; + } + /** * An implementation of {@link ViewOperations} that integrates with {@link IcebergCatalog}. Much * of this code was originally copied from {@link org.apache.iceberg.view.BaseViewOperations}. diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java index 5ce54760a..46d7f5518 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java @@ -112,6 +112,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; import org.apache.polaris.core.exceptions.CommitConflictException; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -2318,37 +2319,43 @@ public abstract class AbstractIcebergCatalogTest extends CatalogTests<IcebergCat .asInstanceOf(InstanceOfAssertFactories.map(String.class, String.class)) .containsEntry(NamespaceEntity.PARENT_NAMESPACE_KEY, NS.toString()) .containsEntry( - "current-snapshot-id", String.valueOf(afterAppend.currentSnapshot().snapshotId())) - .containsEntry("location", afterAppend.location()) - .containsEntry("table-uuid", afterAppend.uuid().toString()) - .containsEntry("current-schema-id", String.valueOf(afterAppend.schema().schemaId())) + IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID, + String.valueOf(afterAppend.currentSnapshot().snapshotId())) + .containsEntry(IcebergTableLikeEntity.LOCATION, afterAppend.location()) + .containsEntry(IcebergTableLikeEntity.TABLE_UUID, afterAppend.uuid().toString()) .containsEntry( - "last-column-id", + IcebergTableLikeEntity.CURRENT_SCHEMA_ID, + String.valueOf(afterAppend.schema().schemaId())) + .containsEntry( + IcebergTableLikeEntity.LAST_COLUMN_ID, afterAppend.schema().columns().stream() .max(Comparator.comparing(Types.NestedField::fieldId)) .map(Types.NestedField::fieldId) .orElse(0) .toString()) .containsEntry( - "last-sequence-number", String.valueOf(afterAppend.currentSnapshot().sequenceNumber())); + IcebergTableLikeEntity.LAST_SEQUENCE_NUMBER, + String.valueOf(afterAppend.currentSnapshot().sequenceNumber())); catalog.loadTable(TABLE).refresh(); catalog.loadTable(TABLE).newFastAppend().appendFile(FILE_B).commit(); validatePropertiesUpdated( schemaResult, - "current-snapshot-id", + IcebergTableLikeEntity.CURRENT_SNAPSHOT_ID, tbl -> String.valueOf(tbl.currentSnapshot().snapshotId())); catalog.loadTable(TABLE).refresh(); catalog.loadTable(TABLE).updateSchema().addColumn("new_col", Types.LongType.get()).commit(); validatePropertiesUpdated( - schemaResult, "current-schema-id", tbl -> String.valueOf(tbl.schema().schemaId())); + schemaResult, + IcebergTableLikeEntity.CURRENT_SCHEMA_ID, + tbl -> String.valueOf(tbl.schema().schemaId())); catalog.loadTable(TABLE).refresh(); catalog.loadTable(TABLE).replaceSortOrder().desc("new_col", NullOrder.NULLS_FIRST).commit(); validatePropertiesUpdated( schemaResult, - "default-sort-order-id", + IcebergTableLikeEntity.DEFAULT_SORT_ORDER_ID, table -> String.valueOf(table.sortOrder().orderId())); }
