Copilot commented on code in PR #2675: URL: https://github.com/apache/iceberg-go/pull/2675#discussion_r4646792019
########## table/timestamp_millis_test.go: ########## @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table_test + +import ( + "context" + "math" + "os" + "path/filepath" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/pqarrow" + parquetschema "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/apache/iceberg-go" + iceio "github.com/apache/iceberg-go/io" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeInt64TimestampParquet( + t *testing.T, + path string, + tsNode parquetschema.Node, + props *parquet.WriterProperties, + ids []int64, + tsVals []int64, +) { + t.Helper() + require.Equal(t, len(ids), len(tsVals)) + + idNode, err := parquetschema.NewPrimitiveNode("id", parquet.Repetitions.Required, parquet.Types.Int64, -1, 1) + require.NoError(t, err) + root, err := parquetschema.NewGroupNode("schema", parquet.Repetitions.Required, + parquetschema.FieldList{idNode, tsNode}, -1) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + f, err := os.Create(path) + require.NoError(t, err) + + pw := file.NewParquetWriter(f, root, file.WithWriterProps(props)) + rgw, err := pw.AppendRowGroupChecked() + require.NoError(t, err) + + cw, err := rgw.NextColumn() + require.NoError(t, err) + _, err = cw.(*file.Int64ColumnChunkWriter).WriteBatch(ids, nil, nil) + require.NoError(t, err) + require.NoError(t, cw.Close()) + + cw, err = rgw.NextColumn() + require.NoError(t, err) + _, err = cw.(*file.Int64ColumnChunkWriter).WriteBatch(tsVals, nil, nil) + require.NoError(t, err) + require.NoError(t, cw.Close()) + + require.NoError(t, rgw.Close()) + require.NoError(t, pw.Close()) Review Comment: `os.Create` returns an `*os.File` that is never closed. `pw.Close()` closes the Parquet writer but does not guarantee the underlying file descriptor is closed; this can leak FDs and cause test flakiness (notably on Windows). Add a `defer` to close `f` (and assert the close error) after successful creation. ########## table/timestamp_millis_test.go: ########## @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table_test + +import ( + "context" + "math" + "os" + "path/filepath" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/pqarrow" + parquetschema "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/apache/iceberg-go" + iceio "github.com/apache/iceberg-go/io" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeInt64TimestampParquet( + t *testing.T, + path string, + tsNode parquetschema.Node, + props *parquet.WriterProperties, + ids []int64, + tsVals []int64, +) { + t.Helper() + require.Equal(t, len(ids), len(tsVals)) + + idNode, err := parquetschema.NewPrimitiveNode("id", parquet.Repetitions.Required, parquet.Types.Int64, -1, 1) + require.NoError(t, err) + root, err := parquetschema.NewGroupNode("schema", parquet.Repetitions.Required, + parquetschema.FieldList{idNode, tsNode}, -1) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + f, err := os.Create(path) + require.NoError(t, err) + + pw := file.NewParquetWriter(f, root, file.WithWriterProps(props)) + rgw, err := pw.AppendRowGroupChecked() + require.NoError(t, err) + + cw, err := rgw.NextColumn() + require.NoError(t, err) + _, err = cw.(*file.Int64ColumnChunkWriter).WriteBatch(ids, nil, nil) + require.NoError(t, err) + require.NoError(t, cw.Close()) + + cw, err = rgw.NextColumn() + require.NoError(t, err) + _, err = cw.(*file.Int64ColumnChunkWriter).WriteBatch(tsVals, nil, nil) + require.NoError(t, err) + require.NoError(t, cw.Close()) + + require.NoError(t, rgw.Close()) + require.NoError(t, pw.Close()) +} + +func newTimestampMillisTestTable(t *testing.T, iceSchema *iceberg.Schema, spec *iceberg.PartitionSpec) *table.Table { + t.Helper() + + location := filepath.ToSlash(t.TempDir()) + meta, err := table.NewMetadata(iceSchema, spec, table.UnsortedSortOrder, location, iceberg.Properties{ + table.PropertyFormatVersion: "2", + }) + require.NoError(t, err) + + return table.New( + table.Identifier{"db", "ts_millis_test"}, + meta, location+"/metadata/v1.metadata.json", + func(ctx context.Context) (iceio.IO, error) { + return iceio.LocalFS{}, nil + }, + &rowDeltaCatalog{metadata: meta}, + ) +} + +func TestAddDataFilesByPathTimestampMillisBoundsAndPartition(t *testing.T) { + t0 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + t1 := t0.Add(1500 * time.Millisecond) + t2 := t0.Add(12 * time.Hour) + const expectedDay = iceberg.Date(19723) // 2024-01-01 in days since 1970-01-01 + + logicalNode := func(unit parquetschema.TimeUnitType, utc bool) parquetschema.Node { + node, err := parquetschema.NewPrimitiveNodeLogical("ts", parquet.Repetitions.Required, + parquetschema.NewTimestampLogicalType(utc, unit), parquet.Types.Int64, -1, 2) + require.NoError(t, err) + + return node + } + convertedNode, err := parquetschema.NewPrimitiveNodeConverted("ts", parquet.Repetitions.Required, + parquet.Types.Int64, parquetschema.ConvertedTypes.TimestampMillis, 0, 0, 0, 2) + require.NoError(t, err) + + tests := []struct { + name string + node parquetschema.Node + icebergTy iceberg.PrimitiveType + perMilli int64 + }{ + {"millis logical timestamp", logicalNode(parquetschema.TimeUnitMillis, false), iceberg.PrimitiveTypes.Timestamp, 1}, + {"millis logical timestamptz", logicalNode(parquetschema.TimeUnitMillis, true), iceberg.PrimitiveTypes.TimestampTz, 1}, + {"millis converted type only", convertedNode, iceberg.PrimitiveTypes.TimestampTz, 1}, + {"micros control timestamp", logicalNode(parquetschema.TimeUnitMicros, false), iceberg.PrimitiveTypes.Timestamp, 1000}, + {"micros control timestamptz", logicalNode(parquetschema.TimeUnitMicros, true), iceberg.PrimitiveTypes.TimestampTz, 1000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer alloc.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), alloc) + + iceSchema := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: tt.icebergTy, Required: true}, + ) + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, + FieldID: 1000, + Name: "ts_day", + Transform: iceberg.DayTransform{}, + }) + tbl := newTimestampMillisTestTable(t, iceSchema, &spec) + + dataPath := filepath.Join(tbl.Location(), "data", "data-001.parquet") + writeInt64TimestampParquet(t, dataPath, tt.node, + parquet.NewWriterProperties(parquet.WithStats(true)), + []int64{1, 2, 3}, + []int64{t0.UnixMilli() * tt.perMilli, t1.UnixMilli() * tt.perMilli, t2.UnixMilli() * tt.perMilli}, + ) + + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFilesByPath(ctx, []string{filepath.ToSlash(dataPath)}, nil, false)) + tbl, err := tx.Commit(ctx) + require.NoError(t, err) + + tasks, err := tbl.Scan().PlanFiles(ctx) + require.NoError(t, err) + require.Len(t, tasks, 1) + + df := tasks[0].File + assert.Equal(t, expectedDay, df.Partition()[1000]) + + loLit, err := iceberg.LiteralFromBytes(tt.icebergTy, df.LowerBoundValues()[2]) + require.NoError(t, err) + hiLit, err := iceberg.LiteralFromBytes(tt.icebergTy, df.UpperBoundValues()[2]) + require.NoError(t, err) + assert.Equal(t, iceberg.Timestamp(t0.UnixMicro()), loLit.(iceberg.TypedLiteral[iceberg.Timestamp]).Value()) + assert.Equal(t, iceberg.Timestamp(t2.UnixMicro()), hiLit.(iceberg.TypedLiteral[iceberg.Timestamp]).Value()) + }) + } + + t.Run("overflow omits manifest bounds", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer alloc.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), alloc) + + iceSchema := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp, Required: true}, + ) + tbl := newTimestampMillisTestTable(t, iceSchema, iceberg.UnpartitionedSpec) + + const hugeMillis = math.MaxInt64/1000 + 1 + dataPath := filepath.Join(tbl.Location(), "data", "overflow.parquet") + writeInt64TimestampParquet(t, dataPath, logicalNode(parquetschema.TimeUnitMillis, false), + parquet.NewWriterProperties(parquet.WithStats(true)), + []int64{1, 2}, + []int64{0, hugeMillis}, + ) + + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFilesByPath(ctx, []string{filepath.ToSlash(dataPath)}, nil, false)) + tbl, err := tx.Commit(ctx) + require.NoError(t, err) + + tasks, err := tbl.Scan().PlanFiles(ctx) + require.NoError(t, err) + require.Len(t, tasks, 1) + + df := tasks[0].File + assert.NotContains(t, df.LowerBoundValues(), 2) + assert.NotContains(t, df.UpperBoundValues(), 2) + assert.Contains(t, df.LowerBoundValues(), 1) + assert.Contains(t, df.UpperBoundValues(), 1) + }) +} + +func writeArrowParquetFile( + t *testing.T, + alloc memory.Allocator, + path string, + arrowSc *arrow.Schema, + rec arrow.RecordBatch, + props *parquet.WriterProperties, +) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + f, err := os.Create(path) + require.NoError(t, err) + + tbl := array.NewTableFromRecords(arrowSc, []arrow.RecordBatch{rec}) + defer tbl.Release() + + require.NoError(t, pqarrow.WriteTable(tbl, f, rec.NumRows(), props, + pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(alloc)))) Review Comment: `f` is not closed after writing. `pqarrow.WriteTable` writes to the provided writer but does not close it; leaving the file open can leak file descriptors and delay flushing on some platforms. Close `f` via `defer` immediately after successful creation (and check the close error). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
