laskoviymishka commented on code in PR #1617:
URL: https://github.com/apache/iceberg-go/pull/1617#discussion_r3756960599
##########
table/equality_delete_reader.go:
##########
@@ -205,23 +337,50 @@ func readEqualityDeleteFile(ctx context.Context, fs
iceio.IO, tableSchema *icebe
}
defer tbl.Release()
+ hasFieldIDs, err := VisitArrowSchema(tbl.Schema(), hasIDs{})
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var fileSchema *iceberg.Schema
+ if !hasFieldIDs {
+ if nameMapping == nil {
+ nameMapping = tableSchema.NameMapping()
+ }
+
+ fileSchema, err = ArrowSchemaToIcebergWithOptions(tbl.Schema(),
ArrowToIcebergOptions{
Review Comment:
On the `!hasFieldIDs` path we convert the delete file's full Arrow schema
through the name mapping, so an extra context column whose physical name isn't
in the mapping fails the whole file. The old lookup only resolved the specific
equality columns, so this is a narrow regression for legacy files that dropped
a column.
Low-probability and not blocking; I'd project down to just the equality
columns before the conversion in a follow-up. wdyt?
##########
table/equality_delete_reader.go:
##########
@@ -46,11 +49,140 @@ type equalityDeleteSet struct {
colNames []string
}
+type arrowFieldRef struct {
+ path []int
+}
+
+type arrowFieldRefsByID map[int][]arrowFieldRef
+
+func equalityFieldLocation(filePath string) string {
+ location := filePath
+ if location == "" {
+ location = "data record"
+ }
+
+ return location
+}
+
+// indexArrowFields derives Arrow child paths from the structurally aligned,
+// ID-resolved Iceberg file schema. It deliberately ignores names: dots in an
+// Iceberg name are literal and must not be interpreted as a path.
+func indexArrowFields(schema *iceberg.Schema) arrowFieldRefsByID {
+ refs := make(arrowFieldRefsByID)
+ if schema == nil {
+ return refs
+ }
+
+ var visit func([]iceberg.NestedField, []int)
+ visit = func(fields []iceberg.NestedField, parentPath []int) {
+ for i, field := range fields {
+ path := append(append([]int(nil), parentPath...), i)
+ refs[field.ID] = append(refs[field.ID],
arrowFieldRef{path: path})
+
+ if nested, ok := field.Type.(*iceberg.StructType); ok {
+ visit(nested.Fields(), path)
+ }
+ }
+ }
+ visit(schema.Fields(), nil)
+
+ return refs
+}
+
+// indexArrowFieldsByMetadata is used for delete files whose Arrow fields carry
+// IDs directly, before any name mapping is needed.
+func indexArrowFieldsByMetadata(schema *arrow.Schema) arrowFieldRefsByID {
+ refs := make(arrowFieldRefsByID)
+ var visit func([]arrow.Field, []int)
+ visit = func(fields []arrow.Field, parentPath []int) {
+ for i, field := range fields {
+ path := append(append([]int(nil), parentPath...), i)
+ if id := getFieldID(field); id != nil {
+ refs[*id] = append(refs[*id],
arrowFieldRef{path: path})
+ }
+
+ if nested, ok := field.Type.(*arrow.StructType); ok {
+ visit(nested.Fields(), path)
+ }
+ }
+ }
+ visit(schema.Fields(), nil)
+
+ return refs
+}
+
+func resolveArrowField(refs arrowFieldRefsByID, fieldID int, fieldName,
filePath string) (arrowFieldRef, error) {
+ matches := refs[fieldID]
+ location := equalityFieldLocation(filePath)
+ if len(matches) == 1 {
+ return matches[0], nil
+ }
+ if len(matches) > 1 {
+ return arrowFieldRef{}, fmt.Errorf("%w: equality field ID %d
(%s) in %s: found %d fields",
+ ErrAmbiguousEqualityColumn, fieldID, fieldName,
location, len(matches))
+ }
+
+ return arrowFieldRef{}, fmt.Errorf("equality field ID %d (%s) not found
in %s", fieldID, fieldName, location)
+}
+
+func arrowArrayAtField(record arrow.RecordBatch, ref arrowFieldRef, fieldID
int, fieldName, filePath string) (arrow.Array, error) {
Review Comment:
`arrowArrayAtField` isn't called anywhere. `makeArrowFieldEncoder` uses
`arrowArraysAtField` directly, so I'd just drop it while we're here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
