chaokunyang commented on code in PR #3901: URL: https://github.com/apache/fory/pull/3901#discussion_r3704755586
########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or Review Comment: Keep Encoder decoding on the trusted-data boundary The repository security model classifies Row Format as a trusted in-memory format unless a runtime explicitly exposes an untrusted API. This sentence promotes Decode and FromRow into that stronger contract, but the implementation does not provide the allocation and work limits required for hostile bytes. Since this package is intended for trusted data, remove the corrupt/untrusted safety promise and document that callers must provide trusted, schema-matched row bytes. ########## go/fory/row/schema_bytes.go: ########## @@ -0,0 +1,351 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "math" + + fory "github.com/apache/fory/go/fory" + "github.com/apache/fory/go/fory/meta" +) + +// Schema wire format, byte-compatible with Java SchemaEncoder: +// +// | version (1 byte) | num_fields (var uint32 small7) | fields... | +// +// Field: | header (1 byte) | name size (varint, only if large) | name +// bytes | type info |. Header bits 0-1 hold the index into +// fieldNameEncodings, bits 2-5 hold nameSize-1 (15 means a varint with +// nameSize-15 follows), bit 6 is the nullable flag. Type info is the +// type id byte, plus precision+scale for DECIMAL, a recursive element +// field for LIST, recursive key and value fields for MAP, and a field +// count plus recursive fields for STRUCT. +const ( + schemaVersion = 1 + fieldNameSizeThreshold = 15 + // Caps readField/readType recursion. + maxSchemaNestingDepth = 64 +) + +// Header bits 0-1 store the INDEX into this table (matching Java +// SchemaEncoder.FIELD_NAME_ENCODINGS order), not the meta.Encoding +// wire values, which differ. +var fieldNameEncodings = [3]meta.Encoding{ + meta.UTF_8, + meta.ALL_TO_LOWER_SPECIAL, + meta.LOWER_UPPER_DIGIT_SPECIAL, +} + +// Stateless after construction, safe for concurrent use. +var ( + fieldNameEncoder = meta.NewEncoder('$', '_') + fieldNameDecoder = meta.NewDecoder('$', '_') +) + +func encodingIndexOf(encoding meta.Encoding) int { + for i, e := range fieldNameEncodings { + if e == encoding { + return i + } + } + return -1 +} + +// SchemaToBytes serializes a schema to the cross-language wire format. +func SchemaToBytes(s *Schema) ([]byte, error) { + buf := fory.NewByteBuffer(nil) + if err := SchemaToBuffer(s, buf); err != nil { + return nil, err + } + return buf.GetByteSlice(0, buf.WriterIndex()), nil +} + +// SchemaToBuffer serializes a schema into an existing buffer. +func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + buf.WriteByte_(schemaVersion) + buf.WriteVarUint32Small7(uint32(s.NumFields())) + for _, f := range s.Fields() { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + return nil +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { + if f.Name == "" { + return fmt.Errorf("row: schema field with empty name") + } + encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) + metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) + if err != nil { + return fmt.Errorf("row: encoding field name %q: %w", f.Name, err) + } + nameBytes := metaString.GetEncodedBytes() + nameSize := len(nameBytes) + encodingIndex := encodingIndexOf(metaString.GetEncoding()) + if encodingIndex < 0 { + return fmt.Errorf("row: field name %q got unsupported encoding %d", f.Name, metaString.GetEncoding()) + } + + header := encodingIndex & 0x03 + bigSize := nameSize > fieldNameSizeThreshold + if bigSize { + header |= fieldNameSizeThreshold << 2 + } else { + header |= (nameSize - 1) << 2 + } + if f.Nullable { + header |= 0x40 + } + buf.WriteByte_(byte(header)) + if bigSize { + buf.WriteVarUint32Small7(uint32(nameSize - fieldNameSizeThreshold)) + } + buf.WriteBinary(nameBytes) + return writeSchemaType(buf, f.Type) +} + +func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { + buf.WriteByte_(byte(dataType.TypeID())) + switch t := dataType.(type) { + case DecimalType: + buf.WriteByte_(byte(t.Precision)) + buf.WriteByte_(byte(t.Scale)) + case *ListType: + return writeSchemaField(buf, t.Elem) + case *MapType: + if err := writeSchemaField(buf, t.Key); err != nil { + return err + } + return writeSchemaField(buf, t.Value) + case *StructType: + buf.WriteVarUint32Small7(uint32(len(t.Fields))) + for _, f := range t.Fields { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + } + return nil +} + +// SchemaFromBytes deserializes a schema from the cross-language wire +// format. It is safe on untrusted input: declared sizes are checked Review Comment: Do not expose schema parsing as untrusted-safe This comment explicitly makes SchemaFromBytes an untrusted deserialization API, which is broader than the trusted-only Row Format boundary. It is also stronger than the implementation: nested struct counts can allocate from the same unread tail at every level. Remove the untrusted-input claim and document trusted schema bytes as the precondition. ########## go/fory/row/row.go: ########## @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Row is a zero-copy reader over one row's bytes. Getters read directly +// from the underlying data without deserializing other fields; nested +// values are sub-slice views, never copies (except String). +// +// Fixed-width getters return the zero value for null fields; use +// IsNullAt to distinguish. Variable-width getters return nil (or "") +// for null fields. An out-of-range field index panics; offsets in +// corrupt or untrusted data may panic on slice bounds, so wrap Review Comment: Keep raw Row views trusted-only Suggesting that callers can make arbitrary untrusted rows safe by wrapping getters with recover is misleading: recover handles ordinary panics, not resource exhaustion, and these zero-copy views do not fully validate the row graph. State that Row and its nested views require trusted, schema-matched bytes instead. ########## go/fory/row/encoder.go: ########## @@ -0,0 +1,490 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "reflect" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Encoder converts values of struct type T to and from row bytes using +// a codec tree compiled once with reflection at construction time. The +// schema is inferred per InferSchema. +// +// An Encoder owns a reusable write buffer and is NOT goroutine-safe; +// create one Encoder per goroutine. Decoding is safe on corrupt or +// untrusted input. +type Encoder[T any] struct { + codec *structCodec + hash int64 + buf *fory.ByteBuffer +} + +func NewEncoder[T any]() (*Encoder[T], error) { + t := reflect.TypeOf((*T)(nil)).Elem() + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("row: Encoder requires a struct type, got %v", t) + } + buf := fory.NewByteBuffer(nil) + codec, err := newStructCodec(t, buf) + if err != nil { + return nil, err + } + return &Encoder[T]{codec: codec, hash: ComputeSchemaHash(codec.schema), buf: buf}, nil +} + +func (e *Encoder[T]) Schema() *Schema { return e.codec.schema } + +// SchemaHash returns the cross-language schema hash used by Encode and +// Decode to detect out-of-sync struct definitions. +func (e *Encoder[T]) SchemaHash() int64 { return e.hash } + +// ToRow serializes v to row bytes (no framing, just the row). +func (e *Encoder[T]) ToRow(v *T) ([]byte, error) { + e.buf.SetWriterIndex(0) + if err := e.codec.writeStruct(reflect.ValueOf(v).Elem()); err != nil { + return nil, err + } + out := make([]byte, e.buf.WriterIndex()) + copy(out, e.buf.GetData()[:e.buf.WriterIndex()]) + return out, nil +} + +// FromRow deserializes row bytes produced with this schema. +func (e *Encoder[T]) FromRow(data []byte) (T, error) { + var out T + err := e.FromRowInto(data, &out) + return out, err +} + +func (e *Encoder[T]) FromRowInto(data []byte, out *T) (err error) { + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("row: corrupt row data: %v", p) + } + }() + return e.codec.readStruct(NewRow(e.codec.schema, data), reflect.ValueOf(out).Elem()) +} + +// Encode frames the row for cross-language exchange: an int64 +// little-endian schema hash followed by the row bytes, matching the +// Java and Python row encoders. +func (e *Encoder[T]) Encode(v *T) ([]byte, error) { + rowBytes, err := e.ToRow(v) + if err != nil { + return nil, err + } + out := make([]byte, 8+len(rowBytes)) + binary.LittleEndian.PutUint64(out, uint64(e.hash)) + copy(out[8:], rowBytes) + return out, nil +} + +// Decode verifies the schema hash and deserializes the row. +func (e *Encoder[T]) Decode(data []byte) (T, error) { + var out T + if len(data) < 8 { + return out, fmt.Errorf("row: encoded data of %d bytes is too short for the schema hash", len(data)) + } + hash := int64(binary.LittleEndian.Uint64(data)) + if hash != e.hash { + return out, fmt.Errorf("row: schema hash mismatch: data has %d, encoder expects %d; writer and reader struct definitions are out of sync", hash, e.hash) + } + err := e.FromRowInto(data[8:], &out) + return out, err +} + +// slotWriter is the shared write surface of RowWriter and ArrayWriter, +// letting one value codec serve struct fields and array elements. +type slotWriter interface { + SetNullAt(i int) + WriteBool(i int, v bool) + WriteInt8(i int, v int8) + WriteInt16(i int, v int16) + WriteInt32(i int, v int32) + WriteInt64(i int, v int64) + WriteFloat32(i int, v float32) + WriteFloat64(i int, v float64) + WriteDate(i int, d fory.Date) error + WriteTimestamp(i int, t time.Time) + WriteDuration(i int, d time.Duration) + WriteString(i int, s string) error + WriteBytes(i int, b []byte) error + SetOffsetAndSize(i, absStart, size int) error +} + +// valueReader is the shared read surface of Row and ArrayData. +type valueReader interface { + IsNullAt(i int) bool + Bool(i int) bool + Int8(i int) int8 + Int16(i int) int16 + Int32(i int) int32 + Int64(i int) int64 + Float32(i int) float32 + Float64(i int) float64 + Date(i int) fory.Date + Timestamp(i int) time.Time + Duration(i int) time.Duration + String(i int) string + Binary(i int) []byte + Struct(i int) *Row + Array(i int) *ArrayData + Map(i int) *MapData +} + +// valueCodec reads or writes one value at slot/element i. Write +// functions handle nil for nilable Go types; read functions overwrite +// the target even when the field is null, so reused targets stay clean. +type valueCodec struct { + write func(w slotWriter, i int, v reflect.Value) error + read func(g valueReader, i int, v reflect.Value) error +} + +// structCodec writes and reads one struct level; nested structs hold +// their own child codec with a child RowWriter sharing the buffer. +type structCodec struct { + schema *Schema + writer *RowWriter + fields []fieldCodec +} + +type fieldCodec struct { + goIndex int + codec valueCodec +} + +func newStructCodec(t reflect.Type, buf *fory.ByteBuffer) (*structCodec, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + sc := &structCodec{ + schema: layout.schema, + writer: NewRowWriterWithBuffer(layout.schema, buf), + fields: make([]fieldCodec, 0, layout.schema.NumFields()), + } + for ordinal, goIndex := range layout.indexes { + codec, err := newValueCodec(t.Field(goIndex).Type, layout.schema.Field(ordinal).Type, buf) Review Comment: Reject null when the Go carrier cannot represent it The codec is built from Field.Type only, so nullability and the target carrier capability are lost. A frame produced for [](*int32){nil} has the same type hash as []int32 and decodes successfully as {0}; nullable strings and value structs similarly become empty values. Pass the Field/nullability into codec construction and return an error when a non-pointer carrier receives null. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + } + elem, err := inferField(listItemName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + case reflect.Map: + if t.Key().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: pointer map key type %v is unsupported", t) + } + key, err := inferField(mapKeyName, t.Key(), path) + if err != nil { + return Field{}, err + } + key.Nullable = false + value, err := inferField(mapValueName, t.Elem(), path) Review Comment: Canonicalize map value nullability during inference For map[string]int32 this leaves the value field non-nullable, while both Java MapType and the Go schema parser canonicalize map values as nullable. As a result, SchemaFromBytes(SchemaToBytes(inferred)) is not equal to the inferred schema and Java emits different schema bytes. Force the canonical map child fields here and add a primitive-valued map round-trip/interoperability case. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { Review Comment: Align []byte with the Java row carrier Go infers []byte as BINARY/raw bytes, but the current Java row TypeInference treats ordinary byte[] as LIST<int8>; only BinaryArray maps to BINARY. Common Go and Java models therefore get different hashes and incompatible raw layouts. Align Java row inference with the shared byte[] mapping, or define and test the exact Java carrier required for this Go type. ########## go/fory/row/schema_bytes.go: ########## @@ -0,0 +1,351 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "math" + + fory "github.com/apache/fory/go/fory" + "github.com/apache/fory/go/fory/meta" +) + +// Schema wire format, byte-compatible with Java SchemaEncoder: +// +// | version (1 byte) | num_fields (var uint32 small7) | fields... | +// +// Field: | header (1 byte) | name size (varint, only if large) | name +// bytes | type info |. Header bits 0-1 hold the index into +// fieldNameEncodings, bits 2-5 hold nameSize-1 (15 means a varint with +// nameSize-15 follows), bit 6 is the nullable flag. Type info is the +// type id byte, plus precision+scale for DECIMAL, a recursive element +// field for LIST, recursive key and value fields for MAP, and a field +// count plus recursive fields for STRUCT. +const ( + schemaVersion = 1 + fieldNameSizeThreshold = 15 + // Caps readField/readType recursion. + maxSchemaNestingDepth = 64 +) + +// Header bits 0-1 store the INDEX into this table (matching Java +// SchemaEncoder.FIELD_NAME_ENCODINGS order), not the meta.Encoding +// wire values, which differ. +var fieldNameEncodings = [3]meta.Encoding{ + meta.UTF_8, + meta.ALL_TO_LOWER_SPECIAL, + meta.LOWER_UPPER_DIGIT_SPECIAL, +} + +// Stateless after construction, safe for concurrent use. +var ( + fieldNameEncoder = meta.NewEncoder('$', '_') + fieldNameDecoder = meta.NewDecoder('$', '_') +) + +func encodingIndexOf(encoding meta.Encoding) int { + for i, e := range fieldNameEncodings { + if e == encoding { + return i + } + } + return -1 +} + +// SchemaToBytes serializes a schema to the cross-language wire format. +func SchemaToBytes(s *Schema) ([]byte, error) { + buf := fory.NewByteBuffer(nil) + if err := SchemaToBuffer(s, buf); err != nil { + return nil, err + } + return buf.GetByteSlice(0, buf.WriterIndex()), nil +} + +// SchemaToBuffer serializes a schema into an existing buffer. +func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + buf.WriteByte_(schemaVersion) + buf.WriteVarUint32Small7(uint32(s.NumFields())) + for _, f := range s.Fields() { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + return nil +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { + if f.Name == "" { + return fmt.Errorf("row: schema field with empty name") + } + encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) + metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) + if err != nil { + return fmt.Errorf("row: encoding field name %q: %w", f.Name, err) + } + nameBytes := metaString.GetEncodedBytes() + nameSize := len(nameBytes) + encodingIndex := encodingIndexOf(metaString.GetEncoding()) + if encodingIndex < 0 { + return fmt.Errorf("row: field name %q got unsupported encoding %d", f.Name, metaString.GetEncoding()) + } + + header := encodingIndex & 0x03 + bigSize := nameSize > fieldNameSizeThreshold + if bigSize { + header |= fieldNameSizeThreshold << 2 + } else { + header |= (nameSize - 1) << 2 + } + if f.Nullable { + header |= 0x40 + } + buf.WriteByte_(byte(header)) + if bigSize { + buf.WriteVarUint32Small7(uint32(nameSize - fieldNameSizeThreshold)) + } + buf.WriteBinary(nameBytes) + return writeSchemaType(buf, f.Type) +} + +func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { + buf.WriteByte_(byte(dataType.TypeID())) + switch t := dataType.(type) { + case DecimalType: + buf.WriteByte_(byte(t.Precision)) + buf.WriteByte_(byte(t.Scale)) + case *ListType: + return writeSchemaField(buf, t.Elem) + case *MapType: + if err := writeSchemaField(buf, t.Key); err != nil { + return err + } + return writeSchemaField(buf, t.Value) + case *StructType: + buf.WriteVarUint32Small7(uint32(len(t.Fields))) + for _, f := range t.Fields { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + } + return nil +} + +// SchemaFromBytes deserializes a schema from the cross-language wire +// format. It is safe on untrusted input: declared sizes are checked +// against remaining bytes before any allocation. +func SchemaFromBytes(data []byte) (*Schema, error) { + r := &schemaReader{buf: fory.NewByteBuffer(data), size: len(data)} + version := r.buf.ReadUint8(&r.err) + if err := r.err.CheckError(); err != nil { + return nil, err + } + if version != schemaVersion { + return nil, fmt.Errorf("row: unsupported schema version %d, expected %d", version, schemaVersion) + } + numFields := int(r.buf.ReadVarUint32Small7(&r.err)) Review Comment: Validate wire integers before narrowing to int On 32-bit Go, converting the uint32 count first can produce a negative int that bypasses the remaining-byte check and reaches make with a negative capacity. Array counts and map key sizes have the same truncation problem, and INT64 decoded into a Go int can silently narrow. Validate against MaxInt before conversion, check reflect.Value.OverflowInt, and make the new tests cross-compile on 386 instead of using untyped 1<<40/1<<33 int arguments. ########## java/fory-format/src/test/java/org/apache/fory/format/GoCrossLanguageTest.java: ########## @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.format; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.fory.format.encoder.Encoders; +import org.apache.fory.format.encoder.RowEncoder; +import org.apache.fory.format.row.binary.BinaryRow; +import org.apache.fory.format.type.DataTypes; +import org.apache.fory.memory.MemoryBuffer; +import org.apache.fory.memory.MemoryUtils; +import org.apache.fory.test.TestUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Row format cross-language tests against a Go peer built from {@code go/fory/tests/row_xlang}. + * Data shapes are shared with {@link CrossLanguageTest} so the Java, Python, and Go peers exercise + * the same schemas. + */ +@Test +public class GoCrossLanguageTest { + private static final boolean IS_WINDOWS = + System.getProperty("os.name").toLowerCase().contains("windows"); + private static final String GO_BINARY = IS_WINDOWS ? "row_xlang_bin.exe" : "row_xlang_bin"; + + @BeforeClass + public void ensureGoPeerReady() { + String enabled = System.getenv("FORY_GO_JAVA_CI"); + if (!"1".equals(enabled)) { + throw new SkipException("Skipping GoCrossLanguageTest: FORY_GO_JAVA_CI not set to 1"); + } + boolean goInstalled = true; + try { + Process process = new ProcessBuilder("go", "version").start(); + if (process.waitFor() != 0) { + goInstalled = false; + } + } catch (IOException | InterruptedException e) { + goInstalled = false; + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + if (!goInstalled) { + throw new SkipException("Skipping GoCrossLanguageTest: go not installed"); + } + List<String> buildCommand = + Arrays.asList("go", "build", "-o", "tests/" + GO_BINARY, "./tests/row_xlang"); + boolean buildSuccess = + TestUtils.executeCommand( + buildCommand, 120, Collections.emptyMap(), new File("../../go/fory")); + if (!buildSuccess || !new File("../../go/fory/tests/" + GO_BINARY).exists()) { + throw new SkipException("Skipping GoCrossLanguageTest: failed to build " + GO_BINARY); Review Comment: Fail the enabled cross-language test when the Go peer cannot build FORY_GO_JAVA_CI=1 explicitly opts into this suite, but a missing Go toolchain, failed go build, or missing binary is still converted to SkipException. The normal Go task does not compile tests/row_xlang, so a peer-only compile regression can leave CI green. Once enabled, every setup failure should fail the test; only the non-opted-in case should skip. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + } + elem, err := inferField(listItemName, t.Elem(), path) Review Comment: Detect recursive slice and map aliases The cycle check is only entered for structs. Valid declarations such as type L []L and type M map[string]M recurse indefinitely through inferField and eventually overflow the stack. Track every composite type in the active path and return the same cycle error used for structs. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
