zeroshade commented on code in PR #144:
URL:
https://github.com/apache/terraform-provider-iceberg/pull/144#discussion_r4620284836
##########
internal/provider/resource_table.go:
##########
@@ -497,121 +505,130 @@ func (r *icebergTableResource)
calculateSchemaUpdates(ctx context.Context, plan,
return nil
}
- // Find the highest existing schema ID to ensure the new one is unique.
- maxSchemaID := int64(0)
- for _, s := range tbl.Metadata().Schemas() {
- if int64(s.ID) > maxSchemaID {
- maxSchemaID = int64(s.ID)
- }
+ if tbl.Schema().Equals(planIceberg) {
+ return nil
}
- newSchemaID := maxSchemaID + 1
- planSchema.ID = types.Int64Value(newSchemaID)
- planIceberg, err = planSchema.ToIceberg()
- if err != nil {
- diags.AddError("failed to convert plan schema with new ID",
err.Error())
-
- return nil
+ // Returning to an earlier schema: point back at its existing ID.
+ maxSchemaID := 0
+ for _, existing := range tbl.Metadata().Schemas() {
+ if existing.Equals(planIceberg) {
Review Comment:
Interaction with #135: there, `validateNewFieldIDs` runs earlier in this
function, so an exact revert to a schema whose dropped column is re-declared
with its old explicit `id` would be rejected before reaching this match, even
though pointing back at that schema is legitimate. Whichever PR lands second
should check for an existing match first, or exempt exact matches from that
validation.
##########
internal/provider/resource_table_test.go:
##########
@@ -1060,3 +1066,350 @@ resource "iceberg_table" "test" {
}
`, tableName, colName)
}
+
+func TestAccIcebergTableRevertToExistingMetadata(t *testing.T) {
+ catalogURI := os.Getenv("ICEBERG_CATALOG_URI")
+ if catalogURI == "" {
+ catalogURI = "http://localhost:8181"
+ }
+
+ providerCfg := fmt.Sprintf(providerConfig, catalogURI)
+ tableName := "revert_test"
+
+ resource.Test(t, resource.TestCase{
+ PreCheck: func() { testAccPreCheck(t) },
+ ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
+ Steps: []resource.TestStep{
+ {
+ Config:
testAccIcebergTableRevertConfig(providerCfg, tableName, false, "asc"),
+ Check: resource.ComposeTestCheckFunc(
+
resource.TestCheckResourceAttr("iceberg_table.test", "schema.id", "0"),
+
resource.TestCheckNoResourceAttr("iceberg_table.test", "partition_spec"),
+
resource.TestCheckResourceAttr("iceberg_table.test", "sort_order.order_id",
"1"),
+ ),
+ },
+ {
+ Config:
testAccIcebergTableRevertConfig(providerCfg, tableName, true, "desc"),
+ Check: resource.ComposeTestCheckFunc(
+
resource.TestCheckResourceAttr("iceberg_table.test", "schema.id", "1"),
+
resource.TestCheckResourceAttr("iceberg_table.test", "partition_spec.spec_id",
"1"),
+
resource.TestCheckResourceAttr("iceberg_table.test", "sort_order.order_id",
"2"),
+ ),
+ },
+ {
+ Config:
testAccIcebergTableRevertConfig(providerCfg, tableName, false, "asc"),
+ Check: resource.ComposeTestCheckFunc(
+
resource.TestCheckResourceAttr("iceberg_table.test", "schema.id", "0"),
+
resource.TestCheckResourceAttr("iceberg_table.test", "schema.fields.#", "2"),
+
resource.TestCheckNoResourceAttr("iceberg_table.test", "partition_spec"),
+
resource.TestCheckResourceAttr("iceberg_table.test", "sort_order.order_id",
"1"),
+ ),
+ },
+ },
+ })
+}
+
+func testAccIcebergTableRevertConfig(providerCfg, tableName string, evolved
bool, sortDirection string) string {
+ extraField, partitionSpec := "", ""
+ if evolved {
+ extraField = `,
+ {
+ name = "b"
+ type = "string"
+ required = false
+ }`
+ partitionSpec = `
+ partition_spec = {
+ fields = [
+ {
+ source_ids = [2]
+ name = "ts_day"
+ transform = "day"
+ }
+ ]
+ }`
+ }
+
+ return providerCfg + fmt.Sprintf(`
+resource "iceberg_namespace" "db_revert" {
+ name = ["db_revert"]
+}
+
+resource "iceberg_table" "test" {
+ namespace = iceberg_namespace.db_revert.name
+ name = "%s"
+ schema = {
+ fields = [
+ {
+ name = "id"
+ type = "long"
+ required = true
+ },
+ {
+ name = "ts"
+ type = "timestamp"
+ required = false
+ }%s
+ ]
+ }%s
+ sort_order = {
+ fields = [
+ {
+ source_id = 1
+ transform = "identity"
+ direction = "%s"
+ null_order = "nulls-first"
+ }
+ ]
+ }
+}
+`, tableName, extraField, partitionSpec, sortDirection)
+}
+
+// historyMetadata is a table whose current schema, spec, and sort order are
+// the last of several, so the earlier ones can be returned to.
+const historyMetadata = `{
+ "format-version": 2,
+ "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
+ "location": "s3://bucket/t",
+ "last-sequence-number": 0,
+ "last-updated-ms": 1602638573590,
+ "last-column-id": 3,
+ "current-schema-id": 1,
+ "schemas": [
+ {"type": "struct", "schema-id": 0, "fields": [
+ {"id": 1, "name": "id", "required": true, "type": "long"},
+ {"id": 2, "name": "ts", "required": false, "type": "timestamp"}
+ ]},
+ {"type": "struct", "schema-id": 1, "fields": [
+ {"id": 1, "name": "id", "required": true, "type": "long"},
+ {"id": 2, "name": "ts", "required": false, "type": "timestamp"},
+ {"id": 3, "name": "b", "required": false, "type": "string"}
+ ]}
+ ],
+ "default-spec-id": 1,
+ "partition-specs": [
+ {"spec-id": 0, "fields": []},
+ {"spec-id": 1, "fields": [{"source-id": 2, "field-id": 1000, "name":
"ts_day", "transform": "day"}]}
+ ],
+ "last-partition-id": 1000,
+ "default-sort-order-id": 2,
+ "sort-orders": [
+ {"order-id": 0, "fields": []},
+ {"order-id": 1, "fields": [{"source-id": 1, "transform": "identity",
"direction": "asc", "null-order": "nulls-first"}]},
+ {"order-id": 2, "fields": [{"source-id": 1, "transform": "identity",
"direction": "desc", "null-order": "nulls-last"}]}
+ ],
+ "properties": {},
+ "snapshots": [],
+ "snapshot-log": [],
+ "metadata-log": []
+}`
+
+func historyTable(t *testing.T) *table.Table {
+ t.Helper()
+ meta, err := table.ParseMetadataString(historyMetadata)
+ if err != nil {
+ t.Fatalf("parse metadata: %v", err)
+ }
+
+ return table.New(table.Identifier{"db", "t"}, meta,
"s3://bucket/t/metadata/v1.json", nil, nil)
+}
+
+func objectFromJSON[T any](t *testing.T, attrTypes func(T) types.Object, raw
string) types.Object {
+ t.Helper()
+ var v T
+ if err := json.Unmarshal([]byte(raw), &v); err != nil {
+ t.Fatalf("unmarshal %s: %v", raw, err)
+ }
+
+ return attrTypes(v)
+}
+
+func schemaObject(t *testing.T, raw string) types.Object {
+ return objectFromJSON(t, func(s icebergTableSchema) types.Object {
+ obj, d := types.ObjectValueFrom(context.Background(),
icebergTableSchema{}.AttrTypes(), s)
+ if d.HasError() {
+ t.Fatalf("schema object: %v", d)
+ }
+
+ return obj
+ }, raw)
+}
+
+func specObject(t *testing.T, raw string) types.Object {
+ return objectFromJSON(t, func(s icebergTablePartitionSpec) types.Object
{
+ s.SpecID = types.Int64Unknown()
+ obj, d := types.ObjectValueFrom(context.Background(),
icebergTablePartitionSpec{}.AttrTypes(), s)
+ if d.HasError() {
+ t.Fatalf("spec object: %v", d)
+ }
+
+ return obj
+ }, raw)
+}
+
+func sortObject(t *testing.T, raw string) types.Object {
+ return objectFromJSON(t, func(s icebergTableSortOrder) types.Object {
+ s.OrderID = types.Int64Unknown()
+ obj, d := types.ObjectValueFrom(context.Background(),
icebergTableSortOrder{}.AttrTypes(), s)
+ if d.HasError() {
+ t.Fatalf("sort order object: %v", d)
+ }
+
+ return obj
+ }, raw)
+}
+
+func updateActions(t *testing.T, updates []table.Update) string {
+ t.Helper()
+ b, err := json.Marshal(updates)
+ if err != nil {
+ t.Fatalf("marshal updates: %v", err)
+ }
+ var raw []map[string]any
+ if err := json.Unmarshal(b, &raw); err != nil {
+ t.Fatalf("unmarshal updates: %v", err)
+ }
+ var out strings.Builder
+ for _, u := range raw {
+ out.WriteString(u["action"].(string))
+ if s, ok := u["schema"].(map[string]any); ok {
+ b, _ := json.Marshal(s["schema-id"])
+ fmt.Fprintf(&out, "(%s)", b)
+ }
+ for _, k := range []string{"schema-id", "spec-id",
"sort-order-id"} {
+ if v, ok := u[k]; ok {
+ b, _ := json.Marshal(v)
+ fmt.Fprintf(&out, "(%s)", b)
+ }
+ }
+ out.WriteString(";")
+ }
+
+ return out.String()
+}
+
+const (
+ schemaV0 = `{"schema-id": 0, "fields": [
+ {"id": 1, "name": "id", "required": true, "type": "long"},
+ {"id": 2, "name": "ts", "required": false, "type": "timestamp"}]}`
+ schemaV1 = `{"schema-id": 1, "fields": [
+ {"id": 1, "name": "id", "required": true, "type": "long"},
+ {"id": 2, "name": "ts", "required": false, "type": "timestamp"},
+ {"id": 3, "name": "b", "required": false, "type": "string"}]}`
+ schemaNew = `{"schema-id": 1, "fields": [
+ {"id": 1, "name": "id", "required": true, "type": "long"},
+ {"id": 2, "name": "ts", "required": false, "type": "timestamp"},
+ {"id": 4, "name": "c", "required": false, "type": "string"}]}`
+)
+
+func TestCalculateSchemaUpdatesReusesExistingSchema(t *testing.T) {
+ tbl := historyTable(t)
+ r := &icebergTableResource{}
+
+ cases := map[string]struct {
+ plan string
+ want string
+ }{
+ "unchanged": {plan: schemaV1, want: ""},
+ "revert to schema 0": {plan: schemaV0, want:
"set-current-schema(0);"},
Review Comment:
Nit: `schemaV0` carries `schema-id: 0`, so this case doesn't show the ID is
ignored. A real plan carries the current ID from state (1 here); using that
would make the test prove it.
##########
internal/provider/resource_table.go:
##########
@@ -497,121 +505,130 @@ func (r *icebergTableResource)
calculateSchemaUpdates(ctx context.Context, plan,
return nil
}
- // Find the highest existing schema ID to ensure the new one is unique.
- maxSchemaID := int64(0)
- for _, s := range tbl.Metadata().Schemas() {
- if int64(s.ID) > maxSchemaID {
- maxSchemaID = int64(s.ID)
- }
+ if tbl.Schema().Equals(planIceberg) {
+ return nil
}
- newSchemaID := maxSchemaID + 1
- planSchema.ID = types.Int64Value(newSchemaID)
- planIceberg, err = planSchema.ToIceberg()
- if err != nil {
- diags.AddError("failed to convert plan schema with new ID",
err.Error())
-
- return nil
+ // Returning to an earlier schema: point back at its existing ID.
+ maxSchemaID := 0
+ for _, existing := range tbl.Metadata().Schemas() {
+ if existing.Equals(planIceberg) {
+ return
[]table.Update{table.NewSetCurrentSchemaUpdate(existing.ID)}
+ }
+ maxSchemaID = max(maxSchemaID, existing.ID)
}
- // Return two events:
- // 1. Add the schema with new schema ID.
- // 2. Set the table schema explicitly to that new ID.
+ // A new schema: add it under the next unused ID and make it current.
+ planIceberg.ID = maxSchemaID + 1
+
return []table.Update{
table.NewAddSchemaUpdate(planIceberg),
- table.NewSetCurrentSchemaUpdate(int(newSchemaID)),
+ table.NewSetCurrentSchemaUpdate(planIceberg.ID),
}
}
func (r *icebergTableResource) calculatePartitionUpdates(ctx context.Context,
plan *icebergTableResourceModel, tbl *table.Table, diags *diag.Diagnostics)
[]table.Update {
- spec := tbl.Spec()
- if plan.PartitionSpec.IsUnknown() {
- if spec.NumFields() > 0 {
- // Create a new unpartitioned spec and set it as default
- unpartitionedSpec := iceberg.NewPartitionSpec()
-
- return []table.Update{
-
table.NewAddPartitionSpecUpdate(&unpartitionedSpec, false),
- table.NewSetDefaultSpecUpdate(-1),
- }
- }
-
- return nil
- }
-
if plan.PartitionSpec.IsNull() {
return nil
}
- var planSpec icebergTablePartitionSpec
- d := plan.PartitionSpec.As(ctx, &planSpec, basetypes.ObjectAsOptions{})
- diags.Append(d...)
+ newIcebergSpec := iceberg.NewPartitionSpec()
+ if !plan.PartitionSpec.IsUnknown() {
+ var planSpec icebergTablePartitionSpec
+ d := plan.PartitionSpec.As(ctx, &planSpec,
basetypes.ObjectAsOptions{})
+ diags.Append(d...)
- if diags.HasError() {
- return nil
- }
+ if diags.HasError() {
+ return nil
+ }
- newIcebergSpec, err := planSpec.ToIceberg()
- if err != nil {
- diags.AddError("failed to convert partition spec", err.Error())
+ spec, err := planSpec.ToIceberg()
+ if err != nil {
+ diags.AddError("failed to convert partition spec",
err.Error())
+
+ return nil
+ }
+ newIcebergSpec = *spec
+ }
+ current := tbl.Spec()
+ if current.CompatibleWith(&newIcebergSpec) {
return nil
}
- // Compare with current spec
- if !spec.CompatibleWith(newIcebergSpec) {
- return []table.Update{
- table.NewAddPartitionSpecUpdate(newIcebergSpec, false),
- table.NewSetDefaultSpecUpdate(-1),
+ // Returning to an earlier spec: point back at its existing ID.
+ for _, existing := range tbl.Metadata().PartitionSpecs() {
+ if existing.CompatibleWith(&newIcebergSpec) {
+ return
[]table.Update{table.NewSetDefaultSpecUpdate(existing.ID())}
}
}
- return nil
+ // A new spec: add it and make it the default.
+ return []table.Update{
+ table.NewAddPartitionSpecUpdate(&newIcebergSpec, false),
+ table.NewSetDefaultSpecUpdate(-1),
+ }
}
func (r *icebergTableResource) calculateSortOrderUpdates(ctx context.Context,
plan *icebergTableResourceModel, tbl *table.Table, diags *diag.Diagnostics)
[]table.Update {
- if plan.SortOrder.IsUnknown() {
- if tbl.SortOrder().OrderID() != 0 {
- // Create a new unsorted order and set it as default
- unsortedOrder := table.UnsortedSortOrder
-
- return []table.Update{
- table.NewAddSortOrderUpdate(&unsortedOrder),
- table.NewSetDefaultSortOrderUpdate(0),
- }
- }
-
- return nil
- }
-
if plan.SortOrder.IsNull() {
return nil
}
- var planOrder icebergTableSortOrder
- d := plan.SortOrder.As(ctx, &planOrder, basetypes.ObjectAsOptions{})
- diags.Append(d...)
+ newIcebergOrder := table.UnsortedSortOrder
+ if !plan.SortOrder.IsUnknown() {
+ var planOrder icebergTableSortOrder
+ d := plan.SortOrder.As(ctx, &planOrder,
basetypes.ObjectAsOptions{})
+ diags.Append(d...)
- if diags.HasError() {
+ if diags.HasError() {
+ return nil
+ }
+
+ order, err := planOrder.ToIceberg()
+ if err != nil {
+ diags.AddError("failed to convert sort order",
err.Error())
+
+ return nil
+ }
+ newIcebergOrder = order
+ }
+
+ if sameSortFields(tbl.SortOrder(), newIcebergOrder) {
return nil
}
- newIcebergOrder, err := planOrder.ToIceberg()
- if err != nil {
- diags.AddError("failed to convert sort order", err.Error())
+ // Returning to an earlier sort order: point back at its existing ID.
+ for _, existing := range tbl.Metadata().SortOrders() {
+ if sameSortFields(existing, newIcebergOrder) {
+ return
[]table.Update{table.NewSetDefaultSortOrderUpdate(existing.OrderID())}
+ }
+ }
- return nil
+ // A new sort order: add it and make it the default.
+ return []table.Update{
+ table.NewAddSortOrderUpdate(&newIcebergOrder),
+ table.NewSetDefaultSortOrderUpdate(-1),
}
+}
- // Compare with current sort order
- if !tbl.SortOrder().Equals(newIcebergOrder) {
- return []table.Update{
- table.NewAddSortOrderUpdate(&newIcebergOrder),
- table.NewSetDefaultSortOrderUpdate(-1),
+// sameSortFields matches the catalog's sort order reuse rule: fields are
+// compared and the order ID is ignored.
+func sameSortFields(a, b table.SortOrder) bool {
Review Comment:
Nit: `aFields := slices.Collect(a.Fields())` would replace the manual loop.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]