This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 0b9fa13f0 fix(table): keep rows where the filter is NULL in
copy-on-write deletes (#2111)
0b9fa13f0 is described below
commit 0b9fa13f031febfb76712a7b2ba3412f3981588b
Author: Enes Keles <[email protected]>
AuthorDate: Tue Oct 13 14:08:31 2026 +0300
fix(table): keep rows where the filter is NULL in copy-on-write deletes
(#2111)
* fix(table): keep rows where the filter is NULL in copy-on-write deletes
Signed-off-by: Enes Keles <[email protected]>
* fix(table): keep NaN rows when row-group pruning the copy-on-write
complement
Signed-off-by: Enes Keles <[email protected]>
* fix(table): keep NULL rows in whole-file deletes and IsNaN deletes on old
files
Signed-off-by: Enes Keles <[email protected]>
* test(table): pin the rewrite path and NotNaN on files without the column
Signed-off-by: Enes Keles <[email protected]>
---------
Signed-off-by: Enes Keles <[email protected]>
---
table/copy_on_write_null_rows_test.go | 360 ++++++++++++++++++++++++++++++++++
table/table.go | 3 +
table/transaction.go | 104 +++++++++-
3 files changed, 465 insertions(+), 2 deletions(-)
diff --git a/table/copy_on_write_null_rows_test.go
b/table/copy_on_write_null_rows_test.go
new file mode 100644
index 000000000..b3ea3a860
--- /dev/null
+++ b/table/copy_on_write_null_rows_test.go
@@ -0,0 +1,360 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table_test
+
+import (
+ "context"
+ "math"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/apache/arrow-go/v18/arrow"
+ "github.com/apache/arrow-go/v18/arrow/array"
+ "github.com/apache/arrow-go/v18/arrow/memory"
+ "github.com/apache/iceberg-go"
+ iceio "github.com/apache/iceberg-go/io"
+ "github.com/apache/iceberg-go/table"
+ "github.com/stretchr/testify/require"
+)
+
+// A copy-on-write delete must remove only the rows the filter is true for.
+// Rows where the filter is NULL (because a referenced column is NULL) don't
+// match it and must survive the rewrite.
+func TestCopyOnWriteDeleteKeepsRowsWhereFilterIsNull(t *testing.T) {
+ age := iceberg.Reference("age")
+ name := iceberg.Reference("name")
+
+ // rows: id=1 {25, "a"}, id=2 {30, NULL}, id=3 {NULL, "c"}, id=4 {40,
"d"}
+ cases := []struct {
+ name string
+ filter iceberg.BooleanExpression
+ want []int64
+ }{
+ {"equal", iceberg.EqualTo(age, int64(30)), []int64{1, 3, 4}},
+ {"greater than", iceberg.GreaterThan(age, int64(35)),
[]int64{1, 2, 3}},
+ {"in", iceberg.IsIn(age, int64(30), int64(40)), []int64{1, 3}},
+ // binding reduces the set to one value and the predicate to
Equal
+ {"in with duplicate values", iceberg.IsIn(age, int64(30),
int64(30)), []int64{1, 3, 4}},
+ {"not in", iceberg.NotIn(age, int64(30), int64(40)), []int64{2,
3, 4}},
+ {"not in with duplicate values", iceberg.NotIn(age, int64(30),
int64(30)), []int64{2, 3}},
+ {"not", iceberg.NewNot(iceberg.EqualTo(age, int64(30))),
[]int64{2, 3}},
+ {"is null", iceberg.IsNull(age), []int64{1, 2, 4}},
+ {"and", iceberg.NewAnd(iceberg.EqualTo(age, int64(30)),
iceberg.EqualTo(name, "c")), []int64{1, 2, 3, 4}},
+ {"or", iceberg.NewOr(iceberg.EqualTo(age, int64(25)),
iceberg.EqualTo(name, "c")), []int64{2, 4}},
+ {"not and", iceberg.NewNot(iceberg.NewAnd(iceberg.EqualTo(age,
int64(30)), iceberg.EqualTo(name, "c"))), []int64{2, 3}},
+ // the literal is outside the file's bounds, so every non-NULL
row
+ // matches and only the NULL row keeps the file from being
deleted whole
+ {"not equal outside bounds", iceberg.NotEqualTo(age,
int64(100)), []int64{3}},
+ {"not outside bounds", iceberg.NewNot(iceberg.EqualTo(age,
int64(100))), []int64{3}},
+ {"not in outside bounds", iceberg.NotIn(age, int64(100),
int64(200)), []int64{3}},
+ }
+
+ for _, version := range []string{"2", "3"} {
+ for _, c := range cases {
+ t.Run("v"+version+"/"+c.name, func(t *testing.T) {
+ ctx := context.Background()
+ tbl := newCopyOnWriteNullRowsTable(t, version)
+
+ txn := tbl.NewTransaction()
+ require.NoError(t, txn.Delete(ctx, c.filter,
nil))
+ tbl, err := txn.Commit(ctx)
+ require.NoError(t, err)
+
+ ids := scanIDs(t, tbl)
+ slices.Sort(ids)
+ require.Equal(t, c.want, ids)
+
+ // the NULL row keeps the file from being
deleted whole, so
+ // the file must have been rewritten rather
than dropped
+ summary :=
tbl.CurrentSnapshot().Summary.Properties
+ require.Equal(t, "1",
summary["deleted-data-files"])
+ require.Equal(t, "1",
summary["added-data-files"])
+ })
+ }
+ }
+}
+
+// A filtered overwrite rewrites matched files through the same path.
+func TestCopyOnWriteOverwriteKeepsRowsWhereFilterIsNull(t *testing.T) {
+ ctx := context.Background()
+ tbl := newCopyOnWriteNullRowsTable(t, "2")
+
+ arrowSchema, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false,
false)
+ require.NoError(t, err)
+ data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema,
[]string{
+ `[{"id":5,"age":30,"name":"e"}]`,
+ })
+ require.NoError(t, err)
+ defer data.Release()
+
+ txn := tbl.NewTransaction()
+ require.NoError(t, txn.OverwriteTable(ctx, data, 1024, nil,
+
table.WithOverwriteFilter(iceberg.EqualTo(iceberg.Reference("age"),
int64(30)))))
+ tbl, err = txn.Commit(ctx)
+ require.NoError(t, err)
+
+ ids := scanIDs(t, tbl)
+ slices.Sort(ids)
+ require.Equal(t, []int64{1, 3, 4, 5}, ids)
+}
+
+// For floating-point columns, NaN compares false rather than NULL, so a delete
+// must remove exactly the rows a scan with the same filter returns.
+func TestCopyOnWriteDeleteMatchesScanWithNaNAndNull(t *testing.T) {
+ x := iceberg.Reference("x")
+
+ cases := []struct {
+ name string
+ filter iceberg.BooleanExpression
+ }{
+ {"less than", iceberg.LessThan(x, 5.0)},
+ {"greater than or equal", iceberg.GreaterThanEqual(x, 5.0)},
+ {"not less than", iceberg.NewNot(iceberg.LessThan(x, 5.0))},
+ {"not equal", iceberg.NotEqualTo(x, 1.0)},
+ {"is nan", iceberg.IsNaN(x)},
+ {"not nan", iceberg.NotNaN(x)},
+ {"not is nan", iceberg.NewNot(iceberg.IsNaN(x))},
+ }
+
+ for _, version := range []string{"2", "3"} {
+ for _, c := range cases {
+ t.Run("v"+version+"/"+c.name, func(t *testing.T) {
+ tbl := newCopyOnWriteNaNTable(t, version,
+ []float64{1, math.NaN(), 10, 0},
[]bool{true, true, true, false})
+ requireDeleteMatchesScan(t, tbl, c.filter)
+ })
+ }
+ }
+}
+
+// Row-group pruning on the rows to keep must not skip a NaN row. Parquet
+// min/max leave NaN out, so with every other value on one side of 5 a pruning
+// filter of x >= 5 for NOT(x < 5) would drop the whole row group.
+func TestCopyOnWriteDeleteKeepsNaNRowsInPrunedRowGroups(t *testing.T) {
+ x := iceberg.Reference("x")
+
+ cases := []struct {
+ name string
+ values []float64
+ filter iceberg.BooleanExpression
+ }{
+ {"less than", []float64{1, math.NaN(), 2}, iceberg.LessThan(x,
5.0)},
+ {"less than or equal", []float64{1, math.NaN(), 2},
iceberg.LessThanEqual(x, 5.0)},
+ {"greater than", []float64{10, math.NaN(), 20},
iceberg.GreaterThan(x, 5.0)},
+ {"greater than or equal", []float64{10, math.NaN(), 20},
iceberg.GreaterThanEqual(x, 5.0)},
+ {"or", []float64{1, math.NaN(), 20},
iceberg.NewOr(iceberg.LessThan(x, 5.0), iceberg.GreaterThan(x, 10.0))},
+ {"and", []float64{1, math.NaN(), 2},
iceberg.NewAnd(iceberg.GreaterThan(x, 0.0), iceberg.LessThan(x, 5.0))},
+ }
+
+ for _, version := range []string{"2", "3"} {
+ for _, c := range cases {
+ t.Run("v"+version+"/"+c.name, func(t *testing.T) {
+ tbl := newCopyOnWriteNaNTable(t, version,
c.values, nil)
+ requireDeleteMatchesScan(t, tbl, c.filter)
+ })
+ }
+ }
+}
+
+// Rows of a file written before the filtered column was added read as NULL
+// and must survive, also for IsNaN, which is false rather than NULL for them.
+// NotNaN is true for a NULL, like in Java's Evaluator, so those rows match it
+// and are deleted along with every other non-NaN row.
+func TestCopyOnWriteDeleteKeepsRowsOfFilesWithoutTheColumn(t *testing.T) {
+ x := iceberg.Reference("x")
+
+ cases := []struct {
+ name string
+ filter iceberg.BooleanExpression
+ want []int64
+ }{
+ {"is nan", iceberg.IsNaN(x), []int64{1, 2, 3, 4, 5}},
+ {"is nan and", iceberg.NewAnd(iceberg.IsNaN(x),
iceberg.GreaterThan(iceberg.Reference("id"), int64(0))), []int64{1, 2, 3, 4,
5}},
+ {"not nan", iceberg.NotNaN(x), []int64{6}},
+ {"not is nan", iceberg.NewNot(iceberg.IsNaN(x)), []int64{6}},
+ {"less than", iceberg.LessThan(x, 5.0), []int64{1, 2, 3, 4, 6}},
+ {"is null", iceberg.IsNull(x), []int64{5, 6}},
+ }
+
+ for _, version := range []string{"2", "3"} {
+ for _, c := range cases {
+ t.Run("v"+version+"/"+c.name, func(t *testing.T) {
+ ctx := context.Background()
+ // ids 1 to 4 are written before x exists
+ tbl := newCopyOnWriteNullRowsTable(t, version)
+
+ txn := tbl.NewTransaction()
+ require.NoError(t, txn.UpdateSchema(true,
false).
+ AddColumn([]string{"x"},
iceberg.PrimitiveTypes.Float64, "", false, nil).Commit())
+ tbl, err := txn.Commit(ctx)
+ require.NoError(t, err)
+
+ arrowSchema, err :=
table.SchemaToArrowSchema(tbl.Schema(), nil, false, false)
+ require.NoError(t, err)
+ bldr :=
array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema)
+ defer bldr.Release()
+
bldr.Field(0).(*array.Int64Builder).AppendValues([]int64{5, 6}, nil)
+
bldr.Field(1).(*array.Int64Builder).AppendValues([]int64{50, 60}, nil)
+
bldr.Field(2).(*array.StringBuilder).AppendValues([]string{"e", "f"}, nil)
+
bldr.Field(3).(*array.Float64Builder).AppendValues([]float64{1, math.NaN()},
nil)
+ rec := bldr.NewRecordBatch()
+ defer rec.Release()
+
+ rdr, err := array.NewRecordReader(rec.Schema(),
[]arrow.RecordBatch{rec})
+ require.NoError(t, err)
+ defer rdr.Release()
+
+ tbl, err = tbl.Append(ctx, rdr, nil)
+ require.NoError(t, err)
+
+ txn = tbl.NewTransaction()
+ require.NoError(t, txn.Delete(ctx, c.filter,
nil))
+ tbl, err = txn.Commit(ctx)
+ require.NoError(t, err)
+
+ ids := scanIDs(t, tbl)
+ slices.Sort(ids)
+ require.Equal(t, c.want, ids)
+ })
+ }
+ }
+}
+
+// requireDeleteMatchesScan deletes filter from tbl and checks that exactly the
+// rows a scan with the same filter returns were removed.
+func requireDeleteMatchesScan(t *testing.T, tbl *table.Table, filter
iceberg.BooleanExpression) {
+ t.Helper()
+ ctx := context.Background()
+
+ before := scanIDs(t, tbl)
+ require.NotEmpty(t, before)
+ matched := scanIDsMatching(t, tbl, filter)
+ want := slices.DeleteFunc(before, func(id int64) bool {
+ return slices.Contains(matched, id)
+ })
+ slices.Sort(want)
+
+ txn := tbl.NewTransaction()
+ require.NoError(t, txn.Delete(ctx, filter, nil))
+ tbl, err := txn.Commit(ctx)
+ require.NoError(t, err)
+
+ ids := scanIDs(t, tbl)
+ slices.Sort(ids)
+ require.Equal(t, want, ids, "scan with the filter matched %v", matched)
+}
+
+func newCopyOnWriteNullRowsTable(t *testing.T, version string) *table.Table {
+ t.Helper()
+
+ location := filepath.ToSlash(t.TempDir())
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type:
iceberg.PrimitiveTypes.Int64, Required: true},
+ iceberg.NestedField{ID: 2, Name: "age", Type:
iceberg.PrimitiveTypes.Int64, Required: false},
+ iceberg.NestedField{ID: 3, Name: "name", Type:
iceberg.PrimitiveTypes.String, Required: false},
+ )
+ meta, err := table.NewMetadata(schema, iceberg.UnpartitionedSpec,
table.UnsortedSortOrder, location,
+ iceberg.Properties{
+ table.PropertyFormatVersion: version,
+ table.WriteDeleteModeKey: table.WriteModeCopyOnWrite,
+ })
+ require.NoError(t, err)
+
+ metaLoc := location + "/metadata/v1.metadata.json"
+ fsF := func(context.Context) (iceio.IO, error) { return
iceio.LocalFS{}, nil }
+ cat := &concurrentTestCatalog{metadata: meta, location: metaLoc, fsF:
fsF}
+ tbl := table.New(table.Identifier{"db", "cow_null_rows"}, meta,
metaLoc, fsF, cat)
+
+ arrowSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64},
+ {Name: "age", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
+ {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+ data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema,
[]string{
+ `[{"id":1,"age":25,"name":"a"},{"id":2,"age":30,"name":null},` +
+
`{"id":3,"age":null,"name":"c"},{"id":4,"age":40,"name":"d"}]`,
+ })
+ require.NoError(t, err)
+ defer data.Release()
+
+ rdr := array.NewTableReader(data, -1)
+ defer rdr.Release()
+
+ tbl, err = tbl.Append(context.Background(), rdr, nil)
+ require.NoError(t, err)
+
+ return tbl
+}
+
+func newCopyOnWriteNaNTable(t *testing.T, version string, values []float64,
valid []bool) *table.Table {
+ t.Helper()
+
+ location := filepath.ToSlash(t.TempDir())
+ schema := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "id", Type:
iceberg.PrimitiveTypes.Int64, Required: true},
+ iceberg.NestedField{ID: 2, Name: "x", Type:
iceberg.PrimitiveTypes.Float64, Required: false},
+ )
+ meta, err := table.NewMetadata(schema, iceberg.UnpartitionedSpec,
table.UnsortedSortOrder, location,
+ iceberg.Properties{
+ table.PropertyFormatVersion: version,
+ table.WriteDeleteModeKey: table.WriteModeCopyOnWrite,
+ })
+ require.NoError(t, err)
+
+ metaLoc := location + "/metadata/v1.metadata.json"
+ fsF := func(context.Context) (iceio.IO, error) { return
iceio.LocalFS{}, nil }
+ cat := &concurrentTestCatalog{metadata: meta, location: metaLoc, fsF:
fsF}
+ tbl := table.New(table.Identifier{"db", "cow_nan_rows"}, meta, metaLoc,
fsF, cat)
+
+ bldr := array.NewRecordBuilder(memory.DefaultAllocator,
arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int64},
+ {Name: "x", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+ }, nil))
+ defer bldr.Release()
+ for i := range values {
+ bldr.Field(0).(*array.Int64Builder).Append(int64(i + 1))
+ }
+ bldr.Field(1).(*array.Float64Builder).AppendValues(values, valid)
+ rec := bldr.NewRecordBatch()
+ defer rec.Release()
+
+ rdr, err := array.NewRecordReader(rec.Schema(),
[]arrow.RecordBatch{rec})
+ require.NoError(t, err)
+ defer rdr.Release()
+
+ tbl, err = tbl.Append(context.Background(), rdr, nil)
+ require.NoError(t, err)
+
+ return tbl
+}
+
+func scanIDsMatching(t *testing.T, tbl *table.Table, filter
iceberg.BooleanExpression) []int64 {
+ t.Helper()
+
+ out, err :=
tbl.Scan(table.WithRowFilter(filter)).ToArrowTable(t.Context())
+ require.NoError(t, err)
+ defer out.Release()
+
+ ids := []int64{}
+ for _, chunk := range out.Column(0).Data().Chunks() {
+ ids = append(ids, chunk.(*array.Int64).Int64Values()...)
+ }
+
+ return ids
+}
diff --git a/table/table.go b/table/table.go
index 9bb3df320..f21e8cba0 100644
--- a/table/table.go
+++ b/table/table.go
@@ -356,6 +356,9 @@ func (t Table) Overwrite(ctx context.Context, rdr
array.RecordReader, snapshotPr
// - Files where some rows match and others don't (partial match) are
rewritten to keep only non-matching rows
// - Files where no rows match the filter are kept unchanged
//
+// A row matches only if the filter is true for it. Rows where the filter
evaluates to NULL, for example
+// `age = 30` on a NULL age, do not match and are kept.
+//
// The filter uses both inclusive and strict metrics evaluators on file
statistics to classify files:
// - Inclusive evaluator identifies candidate files that may contain
matching rows
// - Strict evaluator determines if all rows in a file must match the filter
diff --git a/table/transaction.go b/table/transaction.go
index 9516a3b92..3edadcf89 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -2606,6 +2606,9 @@ func WithDeleteCaseInsensitive() DeleteOption {
// - Files where some rows match and others don't (partial match) are
rewritten to keep only non-matching rows
// - Files where no rows match the filter are kept unchanged
//
+// A row matches only if the filter is true for it. Rows where the filter
evaluates to NULL, for example
+// `age = 30` on a NULL age, do not match and are kept.
+//
// The filter uses both inclusive and strict metrics evaluators on file
statistics to classify files:
// - Inclusive evaluator identifies candidate files that may contain
matching rows
// - Strict evaluator determines if all rows in a file must match the filter
@@ -2733,7 +2736,16 @@ func (t *Transaction)
classifyFilesForFilteredDeletions(ctx context.Context, fs
return nil, nil, nil, fmt.Errorf("failed to create inclusive
metrics evaluator: %w", err)
}
- strictEvaluator, err := newStrictMetricsEvaluator(schema, filter,
caseSensitive, false)
+ // A file is deleted as a whole only if the filter is true for all of
its
+ // rows. The strict evaluator treats predicates as two-valued, so
NotEqual
+ // and NotIn would also count rows where the filter is NULL. Evaluate
the
+ // negation of the rows the rewrite keeps instead, so that both paths
agree.
+ notTrue, err := isNotTrueExpr(filter)
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("failed to build negated
filter: %w", err)
+ }
+
+ strictEvaluator, err := newStrictMetricsEvaluator(schema,
iceberg.NewNot(notTrue), caseSensitive, false)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to create strict
metrics evaluator: %w", err)
}
@@ -2850,7 +2862,10 @@ func (t *Transaction) rewriteFilesWithFilter(ctx
context.Context, fs io.IO, upda
return err
}
- complementFilter := iceberg.NewNot(filter)
+ complementFilter, err := isNotTrueExpr(filter)
+ if err != nil {
+ return fmt.Errorf("failed to build complement filter: %w", err)
+ }
// Bind + convert the complement filter once for the whole rewrite. The
// per-batch filter function is reused across every record batch from
@@ -2888,6 +2903,91 @@ func (t *Transaction) rewriteFilesWithFilter(ctx
context.Context, fs io.IO, upda
return nil
}
+// isNotTrueExpr returns an expression that holds exactly for the rows where
+// filter is not true, i.e. where it is false or NULL. Plain NOT(filter) is not
+// enough: under three-valued logic `v = 5` is NULL when v is NULL, and so is
+// NOT(v = 5), so a copy-on-write rewrite would drop rows that never matched.
+//
+// The result contains no NOT. Row-group pruning pushes a NOT down with
+// RewriteNotExpr, which turns NOT(x < 5) into x >= 5, and that is false for
+// NaN. Negating the predicates here keeps NaN rows in both places.
+func isNotTrueExpr(filter iceberg.BooleanExpression)
(iceberg.BooleanExpression, error) {
+ res, err := iceberg.VisitExpr(filter, isNotTrueVisitor{})
+ if err != nil {
+ return nil, err
+ }
+
+ return res.notTrue, nil
+}
+
+// truthExprs holds two never-NULL expressions for a sub-expression e:
+// notTrue holds iff e is false or NULL, notFalse holds iff e is true or NULL.
+type truthExprs struct {
+ notTrue, notFalse iceberg.BooleanExpression
+}
+
+type isNotTrueVisitor struct{}
+
+func (isNotTrueVisitor) VisitTrue() truthExprs {
+ return truthExprs{notTrue: iceberg.AlwaysFalse{}, notFalse:
iceberg.AlwaysTrue{}}
+}
+
+func (isNotTrueVisitor) VisitFalse() truthExprs {
+ return truthExprs{notTrue: iceberg.AlwaysTrue{}, notFalse:
iceberg.AlwaysFalse{}}
+}
+
+func (isNotTrueVisitor) VisitNot(child truthExprs) truthExprs {
+ return truthExprs{notTrue: child.notFalse, notFalse: child.notTrue}
+}
+
+func (isNotTrueVisitor) VisitAnd(left, right truthExprs) truthExprs {
+ return truthExprs{
+ notTrue: iceberg.NewOr(left.notTrue, right.notTrue),
+ notFalse: iceberg.NewAnd(left.notFalse, right.notFalse),
+ }
+}
+
+func (isNotTrueVisitor) VisitOr(left, right truthExprs) truthExprs {
+ return truthExprs{
+ notTrue: iceberg.NewAnd(left.notTrue, right.notTrue),
+ notFalse: iceberg.NewOr(left.notFalse, right.notFalse),
+ }
+}
+
+func (isNotTrueVisitor) VisitUnbound(pred iceberg.UnboundPredicate) truthExprs
{
+ negated := pred.Negate()
+ switch pred.Op() {
+ case iceberg.OpIsNull, iceberg.OpNotNull, iceberg.OpNotNan:
+ // never evaluates to NULL
+ return truthExprs{notTrue: negated, notFalse: pred}
+ case iceberg.OpIsNan:
+ // is_nan(NULL) is false, so NULL rows survive. The guard also
covers
+ // files written before the column existed, where NotNaN on the
missing
+ // column translates to AlwaysFalse.
+ return truthExprs{notTrue: iceberg.NewOr(negated,
iceberg.IsNull(pred.Term())), notFalse: pred}
+ case iceberg.OpLT, iceberg.OpLTEQ, iceberg.OpGT, iceberg.OpGTEQ:
+ // NaN fails both x < 5 and x >= 5. IsNaN binds to AlwaysFalse
for
+ // non-floating-point terms.
+ negated = iceberg.NewOr(negated, iceberg.IsNaN(pred.Term()))
+ }
+
+ // Any other predicate is NULL when its term is NULL. Guard In and NotIn
+ // too: binding turns a set that ends up with one value into Equal or
+ // NotEqual, which are NULL for a NULL term. A scan with a multi-valued
+ // NotIn still matches NULL rows, see
+ // https://github.com/apache/iceberg-go/issues/2132.
+ isNull := iceberg.IsNull(pred.Term())
+
+ return truthExprs{
+ notTrue: iceberg.NewOr(negated, isNull),
+ notFalse: iceberg.NewOr(pred, isNull),
+ }
+}
+
+func (isNotTrueVisitor) VisitBound(pred iceberg.BoundPredicate) truthExprs {
+ panic(fmt.Errorf("%w: found already bound predicate: %s",
iceberg.ErrInvalidArgument, pred))
+}
+
// rewriteSingleFileArgs bundles the parameters for
[Transaction.rewriteSingleFile]
// — there are several same-typed fields (two filter values, plus a UUID and an
// int) so positional ordering offers no compile-time order protection.
