chaokunyang commented on code in PR #3901: URL: https://github.com/apache/fory/pull/3901#discussion_r3704758656
########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and +// Decode to detect out-of-sync struct definitions. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + e.buf.SetWriterIndex(0) + if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { + return nil, err + } + out := make([]byte, e.buf.WriterIndex()) + copy(out, e.buf.GetData()[:e.buf.WriterIndex()]) + return out, nil +} + +// FromRow deserializes row bytes produced with this schema. +func (e *Encoder[T]) FromRow(data []byte) (T, error) { + var out T + err := e.FromRowInto(data, &out) + return out, err +} + +func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("row: corrupt row data: %v", p) + } + }() + return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + rowBytes, err := e.ToRow(v) + if err != nil { + return nil, err + } + out := make([]byte, 8+len(rowBytes)) + binary.LittleEndian.PutUint64(out, uint64(e.hash)) + copy(out[8:], rowBytes) + return out, nil +} + +// Decode verifies the schema hash and deserializes the row. +func (e *Encoder[T]) Decode(data []byte) (T, error) { + var out T + if len(data) < 8 { + return out, fmt.Errorf("row: encoded data of %d bytes is too short for the schema hash", len(data)) + } + hash := int64(binary.LittleEndian.Uint64(data)) + if hash != e.hash { + return out, fmt.Errorf("row: schema hash mismatch: data has %d, encoder expects %d; writer and reader struct definitions are out of sync", hash, e.hash) + } + err := e.FromRowInto(data[8:], &out) + return out, err +} + +// slotWriter is the shared write surface of RowWriter and ArrayWriter, +// letting one value codec serve struct fields and array elements. +type slotWriter interface { + SetNullAt(i int) + WriteBool(i int, v bool) + WriteInt8(i int, v int8) + WriteInt16(i int, v int16) + WriteInt32(i int, v int32) + WriteInt64(i int, v int64) + WriteFloat32(i int, v float32) + WriteFloat64(i int, v float64) + WriteDate(i int, d fory.Date) error + WriteTimestamp(i int, t time.Time) + WriteDuration(i int, d time.Duration) + WriteString(i int, s string) error + WriteBytes(i int, b []byte) error + SetOffsetAndSize(i, absStart, size int) error +} + +// valueReader is the shared read surface of Row and ArrayData. +type valueReader interface { + IsNullAt(i int) bool + Bool(i int) bool + Int8(i int) int8 + Int16(i int) int16 + Int32(i int) int32 + Int64(i int) int64 + Float32(i int) float32 + Float64(i int) float64 + Date(i int) fory.Date + Timestamp(i int) time.Time + Duration(i int) time.Duration + String(i int) string + Binary(i int) []byte + Struct(i int) *Row + Array(i int) *ArrayData + Map(i int) *MapData +} + +// valueCodec reads or writes one value at slot/element i. Write +// functions handle nil for nilable Go types; read functions overwrite +// the target even when the field is null, so reused targets stay clean. +type valueCodec struct { + write func(w slotWriter, i int, v reflect.Value) error + read func(g valueReader, i int, v reflect.Value) error +} + +// structCodec writes and reads one struct level; nested structs hold +// their own child codec with a child RowWriter sharing the buffer. +type structCodec struct { + schema *Schema + writer *RowWriter + fields []fieldCodec +} + +type fieldCodec struct { + goIndex int + codec valueCodec +} + +func newStructCodec(t reflect.Type, buf *fory.ByteBuffer) (*structCodec, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + sc := &structCodec{ + schema: layout.schema, + writer: NewRowWriterWithBuffer(layout.schema, buf), + fields: make([]fieldCodec, 0, layout.schema.NumFields()), + } + for ordinal, goIndex := range layout.indexes { + codec, err := newValueCodec(t.Field(goIndex).Type, layout.schema.Field(ordinal).Type, buf) + if err != nil { + return nil, err + } + sc.fields = append(sc.fields, fieldCodec{goIndex: goIndex, codec: codec}) + } + return sc, nil +} + +func (sc *structCodec) writeStruct(v reflect.Value) error { + sc.writer.Reset() + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.write(sc.writer, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func (sc *structCodec) readStruct(r *Row, v reflect.Value) error { + for ordinal := range sc.fields { + fc := &sc.fields[ordinal] + if err := fc.codec.read(r, ordinal, v.Field(fc.goIndex)); err != nil { + return err + } + } + return nil +} + +func newValueCodec(goType reflect.Type, dataType DataType, buf *fory.ByteBuffer) (valueCodec, error) { + if goType.Kind() == reflect.Ptr { + elemType := goType.Elem() + inner, err := newValueCodec(elemType, dataType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + return inner.write(w, i, v.Elem()) + }, + read: func(g valueReader, i int, v reflect.Value) error { + if g.IsNullAt(i) { + v.Set(reflect.Zero(goType)) + return nil + } + p := reflect.New(elemType) + if err := inner.read(g, i, p.Elem()); err != nil { + return err + } + v.Set(p) + return nil + }, + }, nil + } + + switch dt := dataType.(type) { + case BoolType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteBool(i, v.Bool()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetBool(g.Bool(i)); return nil }, + }, nil + case Int8Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt8(i, int8(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int8(i))); return nil }, + }, nil + case Int16Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt16(i, int16(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int16(i))); return nil }, + }, nil + case Int32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt32(i, int32(v.Int())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Int32(i))); return nil }, + }, nil + case Int64Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteInt64(i, v.Int()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(g.Int64(i)); return nil }, + }, nil + case Float32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat32(i, float32(v.Float())); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(float64(g.Float32(i))); return nil }, + }, nil + case Float64Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { w.WriteFloat64(i, v.Float()); return nil }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetFloat(g.Float64(i)); return nil }, + }, nil + case Date32Type: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteDate(i, v.Interface().(fory.Date)) }, + read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Date(i))); return nil }, + }, nil + case TimestampType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + w.WriteTimestamp(i, v.Interface().(time.Time)) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { v.Set(reflect.ValueOf(g.Timestamp(i))); return nil }, + }, nil + case DurationType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + w.WriteDuration(i, time.Duration(v.Int())) + return nil + }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetInt(int64(g.Duration(i))); return nil }, + }, nil + case StringType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { return w.WriteString(i, v.String()) }, + read: func(g valueReader, i int, v reflect.Value) error { v.SetString(g.String(i)); return nil }, + }, nil + case BinaryType: + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + return w.WriteBytes(i, v.Bytes()) + }, + read: func(g valueReader, i int, v reflect.Value) error { + b := g.Binary(i) + if b == nil { + v.SetBytes(nil) + return nil + } + // Copy so the decoded struct never aliases the input. + owned := make([]byte, len(b)) + copy(owned, b) + v.SetBytes(owned) + return nil + }, + }, nil + case *ListType: + return newListCodec(goType, dt, buf) + case *MapType: + return newMapCodec(goType, dt, buf) + case *StructType: + child, err := newStructCodec(goType, buf) + if err != nil { + return valueCodec{}, err + } + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + start := child.writer.buf.WriterIndex() + if err := child.writeStruct(v); err != nil { + return err + } + return w.SetOffsetAndSize(i, start, child.writer.buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + nested := g.Struct(i) + if nested == nil { + v.Set(reflect.Zero(goType)) + return nil + } + return child.readStruct(nested, v) + }, + }, nil + default: + return valueCodec{}, fmt.Errorf("row: data type %s is not supported by the encoder", dataType) + } +} + +func newListCodec(goType reflect.Type, listType *ListType, buf *fory.ByteBuffer) (valueCodec, error) { + elemCodec, err := newValueCodec(goType.Elem(), listType.Elem.Type, buf) + if err != nil { + return valueCodec{}, err + } + writer := NewArrayWriter(listType.Elem, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + if err := writer.Reset(n); err != nil { + return err + } + for j := 0; j < n; j++ { + if err := elemCodec.write(writer, j, v.Index(j)); err != nil { + return err + } + } + return w.SetOffsetAndSize(i, start, buf.WriterIndex()-start) + }, + read: func(g valueReader, i int, v reflect.Value) error { + arr := g.Array(i) + if arr == nil { + v.Set(reflect.Zero(goType)) + return nil + } + if err := arr.validateBounds(); err != nil { + return err + } + n := arr.NumElements() + s := reflect.MakeSlice(goType, n, n) + for j := 0; j < n; j++ { + if err := elemCodec.read(arr, j, s.Index(j)); err != nil { + return err + } + } + v.Set(s) + return nil + }, + }, nil +} + +func newMapCodec(goType reflect.Type, mapType *MapType, buf *fory.ByteBuffer) (valueCodec, error) { + keyCodec, err := newValueCodec(goType.Key(), mapType.Key.Type, buf) + if err != nil { + return valueCodec{}, err + } + valueCodecImpl, err := newValueCodec(goType.Elem(), mapType.Value.Type, buf) + if err != nil { + return valueCodec{}, err + } + mapWriter := NewMapWriter(buf) + keysWriter := NewArrayWriter(mapType.Key, buf) + valuesWriter := NewArrayWriter(mapType.Value, buf) + return valueCodec{ + write: func(w slotWriter, i int, v reflect.Value) error { + if v.IsNil() { + w.SetNullAt(i) + return nil + } + n := v.Len() + start := buf.WriterIndex() + mapWriter.Reset() + // Go map iteration order changes between iterations, so + // snapshot entries once to keep keys and values aligned. + keys := make([]reflect.Value, 0, n) Review Comment: Remove per-entry reflection allocations from map codecs The write path retains two []reflect.Value snapshots and MapIter.Key/Value can allocate per entry; the read path separately performs two reflect.New calls per entry. Write keys while taking one typed value snapshot with SetIterValue, and hoist reusable key/value scratch values outside the decode loop. Add an allocation benchmark so this hot path stays linear without 2N temporary objects. ########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and +// Decode to detect out-of-sync struct definitions. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + e.buf.SetWriterIndex(0) + if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { + return nil, err + } + out := make([]byte, e.buf.WriterIndex()) + copy(out, e.buf.GetData()[:e.buf.WriterIndex()]) + return out, nil +} + +// FromRow deserializes row bytes produced with this schema. +func (e *Encoder[T]) FromRow(data []byte) (T, error) { + var out T + err := e.FromRowInto(data, &out) + return out, err +} + +func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("row: corrupt row data: %v", p) + } + }() + return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + rowBytes, err := e.ToRow(v) Review Comment: Avoid copying the complete row twice in Encode Encode calls ToRow, which already allocates and copies the reusable buffer, then allocates the framed result and copies the row again. Reserve or prepend the hash in the encoder buffer and make only the final owned-output copy; add a benchmark for framed encoding. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + } + elem, err := inferField(listItemName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + case reflect.Map: + if t.Key().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: pointer map key type %v is unsupported", t) + } + key, err := inferField(mapKeyName, t.Key(), path) + if err != nil { + return Field{}, err + } + key.Nullable = false + value, err := inferField(mapValueName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &MapType{Key: key, Value: value}, Nullable: true}, nil + case reflect.Struct: + layout, err := inferStructLayout(t, path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &StructType{Fields: layout.schema.Fields()}, Nullable: true}, nil + default: + return Field{}, fmt.Errorf("row: type %v is unsupported in row format", t) + } +} + +// hasIgnoreTag mirrors the ignore semantics of the core fory tag +// parser (parseFieldTag in field_spec.go): a whole tag of "-" or an +// ignore/ignore=true part skips the field, ignore=false keeps it, and +// any other ignore value is an error. Other tag keys are not used by +// the row format. +func hasIgnoreTag(tag string) (bool, error) { + if tag == "-" { + return true, nil + } + ignore := false + for _, part := range strings.Split(tag, ",") { + part = strings.TrimSpace(part) + if part == "ignore" { + ignore = true + } else if strings.HasPrefix(part, "ignore=") { Review Comment: Use the core fory tag grammar This parser does not actually mirror parseFieldTag: the core accepts case-insensitive true/1/yes and false/0/no, trims both sides of '=', and rejects duplicate keys. Here ignore=yes is rejected, ignore = true is ignored, and duplicates silently overwrite. Reuse the core parser or implement the same grammar and update the test that currently treats ignore=yes as invalid. ########## go/fory/row/writer.go: ########## @@ -0,0 +1,381 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +const maxArrayDataBytes = math.MaxInt32 - 15 + +// RowWriter writes one row into a ByteBuffer: null bitmap, fixed 8-byte +// field slots, then the variable data region. All slot and bitmap +// offsets are relative to the row base, so a nested RowWriter can share +// the parent's buffer and produce a self-contained row. +// +// Reset rebases the writer at the buffer's current writer index and must +// be called before writing each row. Writers are not goroutine-safe. +type RowWriter struct { + schema *Schema + buf *fory.ByteBuffer + base int + numFields int + bitmapWidth int + fixedSize int +} + +func NewRowWriter(schema *Schema) *RowWriter { + return NewRowWriterWithBuffer(schema, fory.NewByteBuffer(nil)) +} + +// NewRowWriterWithBuffer shares an existing buffer, e.g. to write a +// nested struct row into its parent's variable data region. +func NewRowWriterWithBuffer(schema *Schema, buf *fory.ByteBuffer) *RowWriter { + n := schema.NumFields() + bitmapWidth := bitmapWidthInBytes(n) + return &RowWriter{ + schema: schema, + buf: buf, + numFields: n, + bitmapWidth: bitmapWidth, + fixedSize: bitmapWidth + n*8, + } +} + +func (w *RowWriter) Schema() *Schema { return w.schema } +func (w *RowWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and zeroes the +// whole fixed region. The buffer reuses dirty capacity after shrinking, +// so zeroing here is what makes null slots and padding deterministic; +// it also lets narrow fixed-width writes store just the value. +func (w *RowWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(w.fixedSize) + data := w.buf.GetData() + clear(data[w.base : w.base+w.fixedSize]) + w.buf.SetWriterIndex(w.base + w.fixedSize) +} + +// Size returns the bytes written for this row so far. +func (w *RowWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// ToBytes returns a view of the row bytes; it stays valid only until the +// buffer is written to or reset again. +func (w *RowWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *RowWriter) slot(i int) int { + if uint(i) >= uint(w.numFields) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, w.numFields)) + } + return w.base + w.bitmapWidth + i*8 +} + +// SetNullAt marks field i null and zeroes its slot. +func (w *RowWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base:w.base+w.bitmapWidth], i) + binary.LittleEndian.PutUint64(data[slot:], 0) +} + +func (w *RowWriter) SetNotNullAt(i int) { + w.slot(i) // bounds check + clearBit(w.buf.GetData()[w.base:w.base+w.bitmapWidth], i) +} + +func (w *RowWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *RowWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *RowWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *RowWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *RowWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *RowWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *RowWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *RowWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *RowWriter) WriteTimestamp(i int, t time.Time) { + w.WriteInt64(i, t.UnixMicro()) +} + +func (w *RowWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +func (w *RowWriter) WriteString(i int, s string) error { + start := appendStringRegion(w.buf, s) Review Comment: Reject strings that are not valid UTF-8 A Go string may contain arbitrary bytes, but the Row Format string type is UTF-8. Writing string([]byte{0xff}) succeeds here; Java decodes it with a replacement character, so the value changes on cross-language round-trip. Validate UTF-8 in both row and array writers and cover nested/list/map strings. ########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and +// Decode to detect out-of-sync struct definitions. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { Review Comment: Return argument errors for nil public inputs ToRow(nil) and Encode(nil) panic at reflect.Value.Elem instead of using their error result. FromRowInto with a nil output is mislabeled as corrupt data and can even succeed for an empty struct; InferSchema(nil) and SchemaToBytes(nil) have analogous behavior. Validate nil at each public entry point and return a direct argument error. ########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and Review Comment: Describe SchemaHash as a type-shape fingerprint The compatible hash folds recursive type IDs only; it does not cover field names, nullability, decimal parameters, or reordering of same-typed fields. It therefore cannot generally detect out-of-sync struct definitions. Keep the wire-compatible algorithm, but narrow this documentation and the mismatch error to the actual type-shape check. ########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { Review Comment: Add the Go Row Format public documentation surface This adds a complete public package, but the specification, xlang guide, README, and Go guide still list only Java/C++/Python and do not explain NewEncoder, ToRow versus Encode, trusted-data scope, nullable carriers, supported types, thread safety, or zero-copy lifetimes. Update the support matrix and add a runnable Go example before exposing the API. ########## go/fory/row/datatype.go: ########## @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "strings" + + fory "github.com/apache/fory/go/fory" +) + +// DataType describes the row-format type of a field. +// +// Type ids are the cross-language Fory type ids shared with the Java, +// C++, and Python row-format implementations; they appear verbatim in +// the serialized schema wire format. +type DataType interface { + TypeID() fory.TypeId + // ByteWidth returns the fixed storage width in bytes, or -1 for + // variable-width types stored through an offset+size slot. + ByteWidth() int + String() string +} + +// Cross-language child field names for matching Java DataTypes. +const ( + listItemName = "item" + mapKeyName = "key" + mapValueName = "value" +) + +type BoolType struct{} + +func (BoolType) TypeID() fory.TypeId { return fory.BOOL } +func (BoolType) ByteWidth() int { return 1 } +func (BoolType) String() string { return "bool" } + +type Int8Type struct{} + +func (Int8Type) TypeID() fory.TypeId { return fory.INT8 } +func (Int8Type) ByteWidth() int { return 1 } +func (Int8Type) String() string { return "int8" } + +type Int16Type struct{} + +func (Int16Type) TypeID() fory.TypeId { return fory.INT16 } +func (Int16Type) ByteWidth() int { return 2 } +func (Int16Type) String() string { return "int16" } + +type Int32Type struct{} + +func (Int32Type) TypeID() fory.TypeId { return fory.INT32 } +func (Int32Type) ByteWidth() int { return 4 } +func (Int32Type) String() string { return "int32" } + +type Int64Type struct{} + +func (Int64Type) TypeID() fory.TypeId { return fory.INT64 } +func (Int64Type) ByteWidth() int { return 8 } +func (Int64Type) String() string { return "int64" } + +// Float16Type exists so schemas received from other languages parse, +// the Go writer, reader, and encoder do not support it yet. +type Float16Type struct{} + +func (Float16Type) TypeID() fory.TypeId { return fory.FLOAT16 } +func (Float16Type) ByteWidth() int { return 2 } +func (Float16Type) String() string { return "float16" } + +type Float32Type struct{} + +func (Float32Type) TypeID() fory.TypeId { return fory.FLOAT32 } +func (Float32Type) ByteWidth() int { return 4 } +func (Float32Type) String() string { return "float32" } + +type Float64Type struct{} + +func (Float64Type) TypeID() fory.TypeId { return fory.FLOAT64 } +func (Float64Type) ByteWidth() int { return 8 } +func (Float64Type) String() string { return "float64" } + +// StringType values are UTF-8 bytes in the variable data region, +// stored as (offset + size) in the fixed slot region. +type StringType struct{} + +func (StringType) TypeID() fory.TypeId { return fory.STRING } +func (StringType) ByteWidth() int { return -1 } +func (StringType) String() string { return "string" } + +type BinaryType struct{} + +func (BinaryType) TypeID() fory.TypeId { return fory.BINARY } +func (BinaryType) ByteWidth() int { return -1 } +func (BinaryType) String() string { return "binary" } + +// Date32Type values are days since the Unix epoch (int32). +type Date32Type struct{} + +func (Date32Type) TypeID() fory.TypeId { return fory.DATE } +func (Date32Type) ByteWidth() int { return 4 } +func (Date32Type) String() string { return "date32" } + +// TimestampType values are microseconds since the Unix epoch (int64). +type TimestampType struct{} + +func (TimestampType) TypeID() fory.TypeId { return fory.TIMESTAMP } +func (TimestampType) ByteWidth() int { return 8 } +func (TimestampType) String() string { return "timestamp" } + +// DurationType values are microseconds (int64). +type DurationType struct{} + +func (DurationType) TypeID() fory.TypeId { return fory.DURATION } +func (DurationType) ByteWidth() int { return 8 } +func (DurationType) String() string { return "duration" } + +// DecimalType exists so schemas received from other languages parse; the +// Go writer, reader, and encoder do not support it yet. Use it as a +// value, not a pointer, so schema equality compares precision and scale. +type DecimalType struct { + Precision int + Scale int +} + +func (DecimalType) TypeID() fory.TypeId { return fory.DECIMAL } +func (DecimalType) ByteWidth() int { return -1 } +func (t DecimalType) String() string { + return fmt.Sprintf("decimal(%d, %d)", t.Precision, t.Scale) +} + +// ListType is a variable-length sequence of Elem values. +type ListType struct { + Elem Field +} + +func (*ListType) TypeID() fory.TypeId { return fory.LIST } +func (*ListType) ByteWidth() int { return -1 } +func (t *ListType) String() string { return "list<" + t.Elem.Type.String() + ">" } + +// MapType stores keys and values as two adjacent arrays. +type MapType struct { + Key Field + Value Field +} + +func (*MapType) TypeID() fory.TypeId { return fory.MAP } +func (*MapType) ByteWidth() int { return -1 } +func (t *MapType) String() string { + return "map<" + t.Key.Type.String() + ", " + t.Value.Type.String() + ">" +} + +// StructType is a nested row with its own bitmap, slots, and variable +// data region. +type StructType struct { + Fields []Field +} + +func (*StructType) TypeID() fory.TypeId { return fory.STRUCT } +func (*StructType) ByteWidth() int { return -1 } +func (t *StructType) String() string { + var sb strings.Builder + sb.WriteString("struct<") + for i, f := range t.Fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} + +// Field is a named, optionally nullable slot in a schema or composite type. +type Field struct { + Name string + Type DataType + Nullable bool +} + +func NewField(name string, dataType DataType, nullable bool) Field { + return Field{Name: name, Type: dataType, Nullable: nullable} +} + +func (f Field) Equal(o Field) bool { + return f.Name == o.Name && f.Nullable == o.Nullable && dataTypeEqual(f.Type, o.Type) +} + +// List returns a list type with the cross-language element field name +// "item" and nullable elements. Build ListType directly for +// non-nullable elements. +func List(elem DataType) *ListType { + return &ListType{Elem: Field{Name: listItemName, Type: elem, Nullable: true}} +} + +// Map returns a map type with the cross-language field names +// "key"/"value". Keys are always non-nullable. +func Map(key, value DataType) *MapType { + return &MapType{ + Key: Field{Name: mapKeyName, Type: key, Nullable: false}, + Value: Field{Name: mapValueName, Type: value, Nullable: true}, + } +} + +func Struct(fields []Field) *StructType { + return &StructType{Fields: fields} +} + +func dataTypeEqual(a, b DataType) bool { + switch at := a.(type) { + case *ListType: + bt, ok := b.(*ListType) + return ok && at.Elem.Equal(bt.Elem) + case *MapType: + bt, ok := b.(*MapType) + return ok && at.Key.Equal(bt.Key) && at.Value.Equal(bt.Value) + case *StructType: + bt, ok := b.(*StructType) + if !ok || len(at.Fields) != len(bt.Fields) { + return false + } + for i := range at.Fields { + if !at.Fields[i].Equal(bt.Fields[i]) { + return false + } + } + return true + default: + // Primitive types are empty structs and DecimalType is a + // comparable value, so interface equality is exact. + return a == b + } +} + +// Schema describes the fields of a top-level row. +type Schema struct { + fields []Field + byName map[string]int +} + +// NewSchema builds a schema from fields in their schema-declared order, +// which fixes the field slot layout. For duplicate names, FieldIndex +// resolves to the first occurrence. +func NewSchema(fields []Field) *Schema { + byName := make(map[string]int, len(fields)) + for i, f := range fields { + if _, ok := byName[f.Name]; !ok { Review Comment: Make duplicate field-name handling cross-language deterministic Go intentionally keeps the first duplicate name, while Java Schema and StructType keep the last. Identical schema bytes can therefore resolve a name to different ordinals across runtimes. Reject duplicate names during schema construction and parsing, or align the lookup rule and test it cross-language. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
