zeroshade commented on code in PR #10: URL: https://github.com/apache/iceberg-terraform/pull/10#discussion_r2789256992
########## internal/provider/resource_table.go: ########## @@ -0,0 +1,564 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "errors" + "strings" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/diag" + "github.com/hashicorp/terraform-plugin-framework/resource" + rscschema "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/objectplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" + "github.com/hashicorp/terraform-plugin-log/tflog" +) + +var ( + _ resource.Resource = &icebergTableResource{} +) + +func NewTableResource() resource.Resource { + return &icebergTableResource{} +} + +type icebergTableResourceModel struct { + ID types.String `tfsdk:"id"` + Namespace types.List `tfsdk:"namespace"` + Name types.String `tfsdk:"name"` + Schema types.Object `tfsdk:"schema"` + UserProperties types.Map `tfsdk:"user_properties"` + ServerProperties types.Map `tfsdk:"server_properties"` +} + +type icebergTableResource struct { + catalog catalog.Catalog + provider *icebergProvider +} + +func (r *icebergTableResource) Metadata(_ context.Context, req resource.MetadataRequest, resp *resource.MetadataResponse) { + resp.TypeName = req.ProviderTypeName + "_table" +} + +func (r *icebergTableResource) Schema(_ context.Context, _ resource.SchemaRequest, resp *resource.SchemaResponse) { + resp.Schema = rscschema.Schema{ + Description: "A resource for managing Iceberg tables.", + Attributes: map[string]rscschema.Attribute{ + "id": rscschema.StringAttribute{ + Computed: true, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.UseStateForUnknown(), + }, + }, + "namespace": rscschema.ListAttribute{ + Description: "The namespace of the table.", + Required: true, + ElementType: types.StringType, + }, + "name": rscschema.StringAttribute{ + Description: "The name of the table.", + Required: true, + }, + "schema": rscschema.SingleNestedAttribute{ + Description: "The schema of the table.", + Required: true, + // TODO: Update schema in place instead of replacement + PlanModifiers: []planmodifier.Object{ + objectplanmodifier.RequiresReplace(), + }, + Attributes: map[string]rscschema.Attribute{ + "id": rscschema.Int64Attribute{ + Description: "The schema ID.", + Optional: true, + Computed: true, + }, + "fields": rscschema.ListNestedAttribute{ + Description: "The fields of the schema.", + Required: true, + NestedObject: rscschema.NestedAttributeObject{ + Attributes: map[string]rscschema.Attribute{ + "id": rscschema.Int64Attribute{ + Description: "The field ID.", + Optional: true, + Computed: true, + }, + "name": rscschema.StringAttribute{ + Description: "The field name.", + Required: true, + }, + "type": rscschema.StringAttribute{ + Description: "The field type.", + Required: true, + }, + "decimal_properties": rscschema.SingleNestedAttribute{ + Description: "Properties for decimal type.", + Optional: true, + Attributes: map[string]rscschema.Attribute{ + "precision": rscschema.Int64Attribute{ + Description: "The precision of the decimal.", + Required: true, + }, + "scale": rscschema.Int64Attribute{ + Description: "The scale of the decimal.", + Required: true, + }, + }, + }, + "fixed_properties": rscschema.SingleNestedAttribute{ + Description: "Properties for fixed type.", + Optional: true, + Attributes: map[string]rscschema.Attribute{ + "length": rscschema.Int64Attribute{ + Description: "The length of the fixed type.", + Required: true, + }, + }, + }, + "list_properties": rscschema.SingleNestedAttribute{ + Description: "Properties for list type.", + Optional: true, + Attributes: map[string]rscschema.Attribute{ + "element_id": rscschema.Int64Attribute{ + Description: "The list element id.", + Required: true, + }, + "element_type": rscschema.StringAttribute{ + Description: "The list element type.", + Required: true, + }, + "element_required": rscschema.BoolAttribute{ + Description: "Whether the list element is required.", + Required: true, + }, + }, + }, + "map_properties": rscschema.SingleNestedAttribute{ + Description: "Properties for map type.", + Optional: true, + Attributes: map[string]rscschema.Attribute{ + "key_id": rscschema.Int64Attribute{ + Description: "The map key id.", + Required: true, + }, + "key_type": rscschema.StringAttribute{ + Description: "The map key type.", + Required: true, + }, + "value_id": rscschema.Int64Attribute{ + Description: "The map value id.", + Required: true, + }, + "value_type": rscschema.StringAttribute{ + Description: "The map value type.", + Required: true, + }, + "value_required": rscschema.BoolAttribute{ + Description: "Whether the map value is required.", + Required: true, + }, + }, + }, + "struct_properties": rscschema.SingleNestedAttribute{ Review Comment: Is there a way to avoid having to duplicate the whole fields definition like this? More to the point, the way this is currently structured how would someone define a struct with more than one level of fields? It doesn't appear there's any way to do something like: `struct<a: struct<b: struct<c: int, d: string>, e: long>, f: struct<g: int>>` in JSON it would be: ```json { "id": 1, "type": "struct", "fields": [ { "id": 2, "name": "a", "type": "struct", "fields": [{ "id": 3, "name": "b", "type": "struct" "fields": [ {"id": 4, "name": "c", "type": "int" }, {"id": 5, "name": "d", "type": "string" } ] }, { "id": 6, "name": "e", "type": "long" }] }, { "id": 7, "name": "f", "type": "struct", "fields": [{ "id": 8, "name": "g", "type": "int" }] } ] } ``` Maybe we can model the terraform structure after the JSON structure and then use `json.Unmarshal`? ########## dev/docker-compose.yml: ########## @@ -13,40 +13,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" Review Comment: why remove this? ########## internal/provider/table_schema.go: ########## @@ -0,0 +1,589 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "errors" + "regexp" + "strconv" + + "github.com/apache/iceberg-go" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" +) + +type icebergTableSchema struct { + ID types.Int64 `tfsdk:"id"` + Fields []types.Object `tfsdk:"fields"` +} + +func (icebergTableSchema) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaField{}.AttrTypes(), + }, + }, + } +} + +type icebergTableSchemaField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` + StructProperties types.Object `tfsdk:"struct_properties"` +} + +func (icebergTableSchemaField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + "struct_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldStructProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaInnerField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` +} + +func (icebergTableSchemaInnerField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaFieldDecimalProperties struct { + Precision types.Int64 `tfsdk:"precision"` + Scale types.Int64 `tfsdk:"scale"` +} + +func (icebergTableSchemaFieldDecimalProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "precision": types.Int64Type, + "scale": types.Int64Type, + } +} + +type icebergTableSchemaFieldFixedProperties struct { + Length types.Int64 `tfsdk:"length"` +} + +func (icebergTableSchemaFieldFixedProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "length": types.Int64Type, + } +} + +type icebergTableSchemaFieldListProperties struct { + ElementID types.Int64 `tfsdk:"element_id"` + ElementType types.String `tfsdk:"element_type"` + ElementRequired types.Bool `tfsdk:"element_required"` +} + +func (icebergTableSchemaFieldListProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "element_id": types.Int64Type, + "element_type": types.StringType, + "element_required": types.BoolType, + } +} + +type icebergTableSchemaFieldMapProperties struct { + KeyID types.Int64 `tfsdk:"key_id"` + KeyType types.String `tfsdk:"key_type"` + ValueID types.Int64 `tfsdk:"value_id"` + ValueType types.String `tfsdk:"value_type"` + ValueRequired types.Bool `tfsdk:"value_required"` +} + +func (icebergTableSchemaFieldMapProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "key_id": types.Int64Type, + "key_type": types.StringType, + "value_id": types.Int64Type, + "value_type": types.StringType, + "value_required": types.BoolType, + } +} + +type icebergTableSchemaFieldStructProperties struct { + Fields types.List `tfsdk:"fields"` +} + +func (icebergTableSchemaFieldStructProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaInnerField{}.AttrTypes(), + }, + }, + } +} + +func terraformToIcebergType(field icebergTableSchemaField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + if field.StructProperties.IsNull() || field.StructProperties.IsUnknown() { + return nil, errors.New("struct_properties must be set for struct type") + } + var structProps icebergTableSchemaFieldStructProperties + if err := field.StructProperties.As(context.Background(), &structProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse struct_properties") + } + + var innerFields []icebergTableSchemaInnerField + if err := structProps.Fields.ElementsAs(context.Background(), &innerFields, false); err.HasError() { + return nil, errors.New("failed to parse struct fields") + } + + nestedFields := make([]iceberg.NestedField, len(innerFields)) + for i, innerField := range innerFields { + typ, err := terraformToIcebergTypeFromInner(innerField) + if err != nil { + return nil, err + } + nestedFields[i] = iceberg.NestedField{ + ID: int(innerField.ID.ValueInt64()), + Name: innerField.Name.ValueString(), + Type: typ, + Required: innerField.Required.ValueBool(), + Doc: innerField.Doc.ValueString(), + } + } + return &iceberg.StructType{ + Fields: nestedFields, + }, nil + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +func terraformToIcebergTypeFromInner(field icebergTableSchemaInnerField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + return nil, errors.New("nested structs are not supported in this version") + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +var ( + decimalRegex = regexp.MustCompile(`^decimal\((\d+),\s*(\d+)\)$`) + fixedRegex = regexp.MustCompile(`^fixed\((\d+)\)$`) +) + +func stringToType(s string) (iceberg.Type, error) { + switch s { + case "boolean": + return iceberg.BooleanType{}, nil + case "int": + return iceberg.Int32Type{}, nil + case "long": + return iceberg.Int64Type{}, nil + case "float": + return iceberg.Float32Type{}, nil + case "double": + return iceberg.Float64Type{}, nil + case "date": + return iceberg.DateType{}, nil + case "time": + return iceberg.TimeType{}, nil + case "timestamp": + return iceberg.TimestampType{}, nil + case "timestamptz": + return iceberg.TimestampTzType{}, nil + case "timestamp_ns": + return iceberg.TimestampNsType{}, nil + case "timestamptz_ns": + return iceberg.TimestampTzNsType{}, nil + case "string": + return iceberg.StringType{}, nil + case "uuid": + return iceberg.UUIDType{}, nil + case "binary": + return iceberg.BinaryType{}, nil + } + + if matches := decimalRegex.FindStringSubmatch(s); matches != nil { + p, _ := strconv.Atoi(matches[1]) + s, _ := strconv.Atoi(matches[2]) + return iceberg.DecimalTypeOf(p, s), nil + } Review Comment: if you're already supporting the string parsing here, why do we need the extra `decimal_properties` object? ########## internal/provider/table_schema.go: ########## @@ -0,0 +1,589 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "errors" + "regexp" + "strconv" + + "github.com/apache/iceberg-go" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" +) + +type icebergTableSchema struct { + ID types.Int64 `tfsdk:"id"` + Fields []types.Object `tfsdk:"fields"` +} + +func (icebergTableSchema) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaField{}.AttrTypes(), + }, + }, + } +} + +type icebergTableSchemaField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` + StructProperties types.Object `tfsdk:"struct_properties"` +} + +func (icebergTableSchemaField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + "struct_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldStructProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaInnerField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` +} + +func (icebergTableSchemaInnerField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaFieldDecimalProperties struct { + Precision types.Int64 `tfsdk:"precision"` + Scale types.Int64 `tfsdk:"scale"` +} + +func (icebergTableSchemaFieldDecimalProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "precision": types.Int64Type, + "scale": types.Int64Type, + } +} + +type icebergTableSchemaFieldFixedProperties struct { + Length types.Int64 `tfsdk:"length"` +} + +func (icebergTableSchemaFieldFixedProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "length": types.Int64Type, + } +} + +type icebergTableSchemaFieldListProperties struct { + ElementID types.Int64 `tfsdk:"element_id"` + ElementType types.String `tfsdk:"element_type"` + ElementRequired types.Bool `tfsdk:"element_required"` +} + +func (icebergTableSchemaFieldListProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "element_id": types.Int64Type, + "element_type": types.StringType, + "element_required": types.BoolType, + } +} + +type icebergTableSchemaFieldMapProperties struct { + KeyID types.Int64 `tfsdk:"key_id"` + KeyType types.String `tfsdk:"key_type"` + ValueID types.Int64 `tfsdk:"value_id"` + ValueType types.String `tfsdk:"value_type"` + ValueRequired types.Bool `tfsdk:"value_required"` +} + +func (icebergTableSchemaFieldMapProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "key_id": types.Int64Type, + "key_type": types.StringType, + "value_id": types.Int64Type, + "value_type": types.StringType, + "value_required": types.BoolType, + } +} + +type icebergTableSchemaFieldStructProperties struct { + Fields types.List `tfsdk:"fields"` +} + +func (icebergTableSchemaFieldStructProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaInnerField{}.AttrTypes(), + }, + }, + } +} + +func terraformToIcebergType(field icebergTableSchemaField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + if field.StructProperties.IsNull() || field.StructProperties.IsUnknown() { + return nil, errors.New("struct_properties must be set for struct type") + } + var structProps icebergTableSchemaFieldStructProperties + if err := field.StructProperties.As(context.Background(), &structProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse struct_properties") + } + + var innerFields []icebergTableSchemaInnerField + if err := structProps.Fields.ElementsAs(context.Background(), &innerFields, false); err.HasError() { + return nil, errors.New("failed to parse struct fields") + } + + nestedFields := make([]iceberg.NestedField, len(innerFields)) + for i, innerField := range innerFields { + typ, err := terraformToIcebergTypeFromInner(innerField) + if err != nil { + return nil, err + } + nestedFields[i] = iceberg.NestedField{ + ID: int(innerField.ID.ValueInt64()), + Name: innerField.Name.ValueString(), + Type: typ, + Required: innerField.Required.ValueBool(), + Doc: innerField.Doc.ValueString(), + } + } + return &iceberg.StructType{ + Fields: nestedFields, + }, nil + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +func terraformToIcebergTypeFromInner(field icebergTableSchemaInnerField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + return nil, errors.New("nested structs are not supported in this version") + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +var ( + decimalRegex = regexp.MustCompile(`^decimal\((\d+),\s*(\d+)\)$`) + fixedRegex = regexp.MustCompile(`^fixed\((\d+)\)$`) +) + +func stringToType(s string) (iceberg.Type, error) { + switch s { + case "boolean": + return iceberg.BooleanType{}, nil + case "int": + return iceberg.Int32Type{}, nil + case "long": + return iceberg.Int64Type{}, nil + case "float": + return iceberg.Float32Type{}, nil + case "double": + return iceberg.Float64Type{}, nil + case "date": + return iceberg.DateType{}, nil + case "time": + return iceberg.TimeType{}, nil + case "timestamp": + return iceberg.TimestampType{}, nil + case "timestamptz": + return iceberg.TimestampTzType{}, nil + case "timestamp_ns": + return iceberg.TimestampNsType{}, nil + case "timestamptz_ns": + return iceberg.TimestampTzNsType{}, nil + case "string": + return iceberg.StringType{}, nil + case "uuid": + return iceberg.UUIDType{}, nil + case "binary": + return iceberg.BinaryType{}, nil + } + + if matches := decimalRegex.FindStringSubmatch(s); matches != nil { + p, _ := strconv.Atoi(matches[1]) + s, _ := strconv.Atoi(matches[2]) + return iceberg.DecimalTypeOf(p, s), nil + } + + if matches := fixedRegex.FindStringSubmatch(s); matches != nil { + l, _ := strconv.Atoi(matches[1]) + return iceberg.FixedTypeOf(l), nil + } Review Comment: same as above, if we're already supporting the string parsing here then why the extra need of "fixed_properties" etc. ########## internal/provider/table_schema.go: ########## @@ -0,0 +1,589 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "errors" + "regexp" + "strconv" + + "github.com/apache/iceberg-go" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" +) + +type icebergTableSchema struct { + ID types.Int64 `tfsdk:"id"` + Fields []types.Object `tfsdk:"fields"` +} + +func (icebergTableSchema) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaField{}.AttrTypes(), + }, + }, + } +} + +type icebergTableSchemaField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` + StructProperties types.Object `tfsdk:"struct_properties"` +} + +func (icebergTableSchemaField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + "struct_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldStructProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaInnerField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` Review Comment: where's the struct_properties? ########## internal/provider/table_schema.go: ########## @@ -0,0 +1,589 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "errors" + "regexp" + "strconv" + + "github.com/apache/iceberg-go" + "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/hashicorp/terraform-plugin-framework/types/basetypes" +) + +type icebergTableSchema struct { + ID types.Int64 `tfsdk:"id"` + Fields []types.Object `tfsdk:"fields"` +} + +func (icebergTableSchema) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaField{}.AttrTypes(), + }, + }, + } +} + +type icebergTableSchemaField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` + StructProperties types.Object `tfsdk:"struct_properties"` +} + +func (icebergTableSchemaField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + "struct_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldStructProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaInnerField struct { + ID types.Int64 `tfsdk:"id"` + Name types.String `tfsdk:"name"` + Type types.String `tfsdk:"type"` + Required types.Bool `tfsdk:"required"` + Doc types.String `tfsdk:"doc"` + DecimalProperties types.Object `tfsdk:"decimal_properties"` + FixedProperties types.Object `tfsdk:"fixed_properties"` + ListProperties types.Object `tfsdk:"list_properties"` + MapProperties types.Object `tfsdk:"map_properties"` +} + +func (icebergTableSchemaInnerField) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "id": types.Int64Type, + "name": types.StringType, + "type": types.StringType, + "required": types.BoolType, + "doc": types.StringType, + "decimal_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldDecimalProperties{}.AttrTypes()}, + "fixed_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldFixedProperties{}.AttrTypes()}, + "list_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldListProperties{}.AttrTypes()}, + "map_properties": types.ObjectType{AttrTypes: icebergTableSchemaFieldMapProperties{}.AttrTypes()}, + } +} + +type icebergTableSchemaFieldDecimalProperties struct { + Precision types.Int64 `tfsdk:"precision"` + Scale types.Int64 `tfsdk:"scale"` +} + +func (icebergTableSchemaFieldDecimalProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "precision": types.Int64Type, + "scale": types.Int64Type, + } +} + +type icebergTableSchemaFieldFixedProperties struct { + Length types.Int64 `tfsdk:"length"` +} + +func (icebergTableSchemaFieldFixedProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "length": types.Int64Type, + } +} + +type icebergTableSchemaFieldListProperties struct { + ElementID types.Int64 `tfsdk:"element_id"` + ElementType types.String `tfsdk:"element_type"` + ElementRequired types.Bool `tfsdk:"element_required"` +} + +func (icebergTableSchemaFieldListProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "element_id": types.Int64Type, + "element_type": types.StringType, + "element_required": types.BoolType, + } +} + +type icebergTableSchemaFieldMapProperties struct { + KeyID types.Int64 `tfsdk:"key_id"` + KeyType types.String `tfsdk:"key_type"` + ValueID types.Int64 `tfsdk:"value_id"` + ValueType types.String `tfsdk:"value_type"` + ValueRequired types.Bool `tfsdk:"value_required"` +} + +func (icebergTableSchemaFieldMapProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "key_id": types.Int64Type, + "key_type": types.StringType, + "value_id": types.Int64Type, + "value_type": types.StringType, + "value_required": types.BoolType, + } +} + +type icebergTableSchemaFieldStructProperties struct { + Fields types.List `tfsdk:"fields"` +} + +func (icebergTableSchemaFieldStructProperties) AttrTypes() map[string]attr.Type { + return map[string]attr.Type{ + "fields": types.ListType{ + ElemType: types.ObjectType{ + AttrTypes: icebergTableSchemaInnerField{}.AttrTypes(), + }, + }, + } +} + +func terraformToIcebergType(field icebergTableSchemaField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + if field.StructProperties.IsNull() || field.StructProperties.IsUnknown() { + return nil, errors.New("struct_properties must be set for struct type") + } + var structProps icebergTableSchemaFieldStructProperties + if err := field.StructProperties.As(context.Background(), &structProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse struct_properties") + } + + var innerFields []icebergTableSchemaInnerField + if err := structProps.Fields.ElementsAs(context.Background(), &innerFields, false); err.HasError() { + return nil, errors.New("failed to parse struct fields") + } + + nestedFields := make([]iceberg.NestedField, len(innerFields)) + for i, innerField := range innerFields { + typ, err := terraformToIcebergTypeFromInner(innerField) + if err != nil { + return nil, err + } + nestedFields[i] = iceberg.NestedField{ + ID: int(innerField.ID.ValueInt64()), + Name: innerField.Name.ValueString(), + Type: typ, + Required: innerField.Required.ValueBool(), + Doc: innerField.Doc.ValueString(), + } + } + return &iceberg.StructType{ + Fields: nestedFields, + }, nil + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +func terraformToIcebergTypeFromInner(field icebergTableSchemaInnerField) (iceberg.Type, error) { + typeStr := field.Type.ValueString() + switch typeStr { + case "list": + if field.ListProperties.IsNull() || field.ListProperties.IsUnknown() { + return nil, errors.New("list_properties must be set for list type") + } + var listProps icebergTableSchemaFieldListProperties + if err := field.ListProperties.As(context.Background(), &listProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse list_properties") + } + elemType, err := stringToType(listProps.ElementType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.ListType{ + ElementID: int(listProps.ElementID.ValueInt64()), + Element: elemType, + ElementRequired: listProps.ElementRequired.ValueBool(), + }, nil + + case "map": + if field.MapProperties.IsNull() || field.MapProperties.IsUnknown() { + return nil, errors.New("map_properties must be set for map type") + } + var mapProps icebergTableSchemaFieldMapProperties + if err := field.MapProperties.As(context.Background(), &mapProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse map_properties") + } + keyType, err := stringToType(mapProps.KeyType.ValueString()) + if err != nil { + return nil, err + } + valueType, err := stringToType(mapProps.ValueType.ValueString()) + if err != nil { + return nil, err + } + return &iceberg.MapType{ + KeyID: int(mapProps.KeyID.ValueInt64()), + KeyType: keyType, + ValueID: int(mapProps.ValueID.ValueInt64()), + ValueType: valueType, + ValueRequired: mapProps.ValueRequired.ValueBool(), + }, nil + + case "struct": + return nil, errors.New("nested structs are not supported in this version") + + case "decimal": + if field.DecimalProperties.IsNull() || field.DecimalProperties.IsUnknown() { + return nil, errors.New("decimal_properties must be set for decimal type") + } + var decProps icebergTableSchemaFieldDecimalProperties + if err := field.DecimalProperties.As(context.Background(), &decProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse decimal_properties") + } + return iceberg.DecimalTypeOf( + int(decProps.Precision.ValueInt64()), + int(decProps.Scale.ValueInt64()), + ), nil + + case "fixed": + if field.FixedProperties.IsNull() || field.FixedProperties.IsUnknown() { + return nil, errors.New("fixed_properties must be set for fixed type") + } + var fixedProps icebergTableSchemaFieldFixedProperties + if err := field.FixedProperties.As(context.Background(), &fixedProps, basetypes.ObjectAsOptions{}); err.HasError() { + return nil, errors.New("failed to parse fixed_properties") + } + return iceberg.FixedTypeOf( + int(fixedProps.Length.ValueInt64()), + ), nil + } + + return stringToType(typeStr) +} + +var ( + decimalRegex = regexp.MustCompile(`^decimal\((\d+),\s*(\d+)\)$`) + fixedRegex = regexp.MustCompile(`^fixed\((\d+)\)$`) +) + +func stringToType(s string) (iceberg.Type, error) { + switch s { + case "boolean": + return iceberg.BooleanType{}, nil + case "int": + return iceberg.Int32Type{}, nil + case "long": + return iceberg.Int64Type{}, nil + case "float": + return iceberg.Float32Type{}, nil + case "double": + return iceberg.Float64Type{}, nil + case "date": + return iceberg.DateType{}, nil + case "time": + return iceberg.TimeType{}, nil + case "timestamp": + return iceberg.TimestampType{}, nil + case "timestamptz": + return iceberg.TimestampTzType{}, nil + case "timestamp_ns": + return iceberg.TimestampNsType{}, nil + case "timestamptz_ns": + return iceberg.TimestampTzNsType{}, nil + case "string": + return iceberg.StringType{}, nil + case "uuid": + return iceberg.UUIDType{}, nil + case "binary": + return iceberg.BinaryType{}, nil + } + + if matches := decimalRegex.FindStringSubmatch(s); matches != nil { + p, _ := strconv.Atoi(matches[1]) + s, _ := strconv.Atoi(matches[2]) + return iceberg.DecimalTypeOf(p, s), nil + } + + if matches := fixedRegex.FindStringSubmatch(s); matches != nil { + l, _ := strconv.Atoi(matches[1]) + return iceberg.FixedTypeOf(l), nil + } + + return nil, errors.New("unsupported type: " + s) +} + +func typeToString(t iceberg.Type) (string, error) { + return t.String(), nil +} + +func icebergToTerraformField(field iceberg.NestedField) (attr.Value, error) { + // Defaults + typeStr := field.Type.String() + decimalProps := types.ObjectNull(icebergTableSchemaFieldDecimalProperties{}.AttrTypes()) + fixedProps := types.ObjectNull(icebergTableSchemaFieldFixedProperties{}.AttrTypes()) + listProps := types.ObjectNull(icebergTableSchemaFieldListProperties{}.AttrTypes()) + mapProps := types.ObjectNull(icebergTableSchemaFieldMapProperties{}.AttrTypes()) + structProps := types.ObjectNull(icebergTableSchemaFieldStructProperties{}.AttrTypes()) + + switch t := field.Type.(type) { + case iceberg.DecimalType: + typeStr = "decimal" + decimalProps = types.ObjectValueMust( + icebergTableSchemaFieldDecimalProperties{}.AttrTypes(), + map[string]attr.Value{ + "precision": types.Int64Value(int64(t.Precision())), + "scale": types.Int64Value(int64(t.Scale())), + }, + ) + case iceberg.FixedType: + typeStr = "fixed" + fixedProps = types.ObjectValueMust( + icebergTableSchemaFieldFixedProperties{}.AttrTypes(), + map[string]attr.Value{ + "length": types.Int64Value(int64(t.Len())), + }, + ) + case *iceberg.ListType: + typeStr = "list" + elemTypeStr, err := typeToString(t.Element) + if err != nil { + return nil, err + } + listProps = types.ObjectValueMust( + icebergTableSchemaFieldListProperties{}.AttrTypes(), + map[string]attr.Value{ + "element_id": types.Int64Value(int64(t.ElementID)), + "element_type": types.StringValue(elemTypeStr), + "element_required": types.BoolValue(t.ElementRequired), + }, + ) + case *iceberg.MapType: + typeStr = "map" + keyTypeStr, err := typeToString(t.KeyType) + if err != nil { + return nil, err + } + valueTypeStr, err := typeToString(t.ValueType) + if err != nil { + return nil, err + } + mapProps = types.ObjectValueMust( + icebergTableSchemaFieldMapProperties{}.AttrTypes(), + map[string]attr.Value{ + "key_id": types.Int64Value(int64(t.KeyID)), + "key_type": types.StringValue(keyTypeStr), + "value_id": types.Int64Value(int64(t.ValueID)), + "value_type": types.StringValue(valueTypeStr), + "value_required": types.BoolValue(t.ValueRequired), + }, + ) + case *iceberg.StructType: + typeStr = "struct" + nestedFields := make([]attr.Value, len(t.Fields)) + for i, nestedField := range t.Fields { + f, err := icebergToTerraformInnerField(nestedField) + if err != nil { + return nil, err + } + nestedFields[i] = f + } + structProps = types.ObjectValueMust( + icebergTableSchemaFieldStructProperties{}.AttrTypes(), + map[string]attr.Value{ + "fields": types.ListValueMust(types.ObjectType{AttrTypes: icebergTableSchemaInnerField{}.AttrTypes()}, nestedFields), + }, + ) + } + + doc := types.StringValue(field.Doc) + if field.Doc == "" { + doc = types.StringNull() + } + + return types.ObjectValueMust( + icebergTableSchemaField{}.AttrTypes(), + map[string]attr.Value{ + "id": types.Int64Value(int64(field.ID)), + "name": types.StringValue(field.Name), + "type": types.StringValue(typeStr), + "required": types.BoolValue(field.Required), + "doc": doc, + "decimal_properties": decimalProps, + "fixed_properties": fixedProps, + "list_properties": listProps, + "map_properties": mapProps, + "struct_properties": structProps, + }, + ), nil +} + +func icebergToTerraformInnerField(field iceberg.NestedField) (attr.Value, error) { + typeStr := field.Type.String() + decimalProps := types.ObjectNull(icebergTableSchemaFieldDecimalProperties{}.AttrTypes()) + fixedProps := types.ObjectNull(icebergTableSchemaFieldFixedProperties{}.AttrTypes()) + listProps := types.ObjectNull(icebergTableSchemaFieldListProperties{}.AttrTypes()) + mapProps := types.ObjectNull(icebergTableSchemaFieldMapProperties{}.AttrTypes()) + + switch t := field.Type.(type) { + case iceberg.DecimalType: + typeStr = "decimal" + decimalProps = types.ObjectValueMust( + icebergTableSchemaFieldDecimalProperties{}.AttrTypes(), + map[string]attr.Value{ + "precision": types.Int64Value(int64(t.Precision())), + "scale": types.Int64Value(int64(t.Scale())), + }, + ) + case iceberg.FixedType: + typeStr = "fixed" + fixedProps = types.ObjectValueMust( + icebergTableSchemaFieldFixedProperties{}.AttrTypes(), + map[string]attr.Value{ + "length": types.Int64Value(int64(t.Len())), + }, + ) + case *iceberg.ListType: + typeStr = "list" + elemTypeStr, err := typeToString(t.Element) + if err != nil { + return nil, err + } + listProps = types.ObjectValueMust( + icebergTableSchemaFieldListProperties{}.AttrTypes(), + map[string]attr.Value{ + "element_id": types.Int64Value(int64(t.ElementID)), + "element_type": types.StringValue(elemTypeStr), + "element_required": types.BoolValue(t.ElementRequired), + }, + ) + case *iceberg.MapType: + typeStr = "map" + keyTypeStr, err := typeToString(t.KeyType) + if err != nil { + return nil, err + } + valueTypeStr, err := typeToString(t.ValueType) + if err != nil { + return nil, err + } + mapProps = types.ObjectValueMust( + icebergTableSchemaFieldMapProperties{}.AttrTypes(), + map[string]attr.Value{ + "key_id": types.Int64Value(int64(t.KeyID)), + "key_type": types.StringValue(keyTypeStr), + "value_id": types.Int64Value(int64(t.ValueID)), + "value_type": types.StringValue(valueTypeStr), + "value_required": types.BoolValue(t.ValueRequired), + }, + ) + case *iceberg.StructType: Review Comment: this seems to be the only difference between `icebergToTerraformField` and `icebergToTerraformInnerField` it would be much better to just be truly recursive instead and not have to duplicate the logic between them -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
