chaokunyang commented on code in PR #3901: URL: https://github.com/apache/fory/pull/3901#discussion_r3704757339
########## go/fory/row/row.go: ########## @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +// Row is a zero-copy reader over one row's bytes. Getters read directly +// from the underlying data without deserializing other fields; nested +// values are sub-slice views, never copies (except String). +// +// Fixed-width getters return the zero value for null fields; use +// IsNullAt to distinguish. Variable-width getters return nil (or "") +// for null fields. An out-of-range field index panics; offsets in +// corrupt or untrusted data may panic on slice bounds, so wrap +// untrusted decoding with recover. +type Row struct { + fields []Field + data []byte + bitmapWidth int +} + +func NewRow(schema *Schema, data []byte) *Row { + return newRow(schema.Fields(), data) +} + +func newRow(fields []Field, data []byte) *Row { + return &Row{fields: fields, data: data, bitmapWidth: bitmapWidthInBytes(len(fields))} +} + +func (r *Row) NumFields() int { return len(r.fields) } +func (r *Row) SizeBytes() int { return len(r.data) } + +// Data returns the underlying row bytes; callers must not modify them. +func (r *Row) Data() []byte { return r.data } + +func (r *Row) IsNullAt(i int) bool { + if uint(i) >= uint(len(r.fields)) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, len(r.fields))) + } + return getBit(r.data, i) +} + +func (r *Row) slot(i int) int { return r.bitmapWidth + i*8 } + +func (r *Row) Bool(i int) bool { + return !r.IsNullAt(i) && r.data[r.slot(i)] != 0 +} + +func (r *Row) Int8(i int) int8 { + if r.IsNullAt(i) { + return 0 + } + return int8(r.data[r.slot(i)]) +} + +func (r *Row) Int16(i int) int16 { + if r.IsNullAt(i) { + return 0 + } + return int16(binary.LittleEndian.Uint16(r.data[r.slot(i):])) +} + +func (r *Row) Int32(i int) int32 { + if r.IsNullAt(i) { + return 0 + } + return int32(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Int64(i int) int64 { + if r.IsNullAt(i) { + return 0 + } + return int64(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Float32(i int) float32 { + if r.IsNullAt(i) { + return 0 + } + return math.Float32frombits(binary.LittleEndian.Uint32(r.data[r.slot(i):])) +} + +func (r *Row) Float64(i int) float64 { + if r.IsNullAt(i) { + return 0 + } + return math.Float64frombits(binary.LittleEndian.Uint64(r.data[r.slot(i):])) +} + +func (r *Row) Date(i int) fory.Date { + if r.IsNullAt(i) { + return fory.Date{} + } + return dateFromDays(int32(binary.LittleEndian.Uint32(r.data[r.slot(i):]))) +} + +func (r *Row) Timestamp(i int) time.Time { + if r.IsNullAt(i) { + return time.Time{} + } + return time.UnixMicro(r.Int64(i)) +} + +func (r *Row) Duration(i int) time.Duration { + return time.Duration(r.Int64(i)) * time.Microsecond Review Comment: Check the duration carrier range before multiplying The wire stores an int64 number of microseconds, but time.Duration stores int64 nanoseconds. Multiplication silently wraps for otherwise valid wire values; MaxInt64 microseconds becomes negative. Reject values outside MinInt64/1000 through MaxInt64/1000 before conversion, including the ArrayData path. ########## go/fory/row/writer.go: ########## @@ -0,0 +1,381 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +const maxArrayDataBytes = math.MaxInt32 - 15 + +// RowWriter writes one row into a ByteBuffer: null bitmap, fixed 8-byte +// field slots, then the variable data region. All slot and bitmap +// offsets are relative to the row base, so a nested RowWriter can share +// the parent's buffer and produce a self-contained row. +// +// Reset rebases the writer at the buffer's current writer index and must +// be called before writing each row. Writers are not goroutine-safe. +type RowWriter struct { + schema *Schema + buf *fory.ByteBuffer + base int + numFields int + bitmapWidth int + fixedSize int +} + +func NewRowWriter(schema *Schema) *RowWriter { + return NewRowWriterWithBuffer(schema, fory.NewByteBuffer(nil)) +} + +// NewRowWriterWithBuffer shares an existing buffer, e.g. to write a +// nested struct row into its parent's variable data region. +func NewRowWriterWithBuffer(schema *Schema, buf *fory.ByteBuffer) *RowWriter { + n := schema.NumFields() + bitmapWidth := bitmapWidthInBytes(n) + return &RowWriter{ + schema: schema, + buf: buf, + numFields: n, + bitmapWidth: bitmapWidth, + fixedSize: bitmapWidth + n*8, + } +} + +func (w *RowWriter) Schema() *Schema { return w.schema } +func (w *RowWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and zeroes the +// whole fixed region. The buffer reuses dirty capacity after shrinking, +// so zeroing here is what makes null slots and padding deterministic; +// it also lets narrow fixed-width writes store just the value. +func (w *RowWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(w.fixedSize) + data := w.buf.GetData() + clear(data[w.base : w.base+w.fixedSize]) + w.buf.SetWriterIndex(w.base + w.fixedSize) +} + +// Size returns the bytes written for this row so far. +func (w *RowWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// ToBytes returns a view of the row bytes; it stays valid only until the +// buffer is written to or reset again. +func (w *RowWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *RowWriter) slot(i int) int { + if uint(i) >= uint(w.numFields) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, w.numFields)) + } + return w.base + w.bitmapWidth + i*8 +} + +// SetNullAt marks field i null and zeroes its slot. +func (w *RowWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base:w.base+w.bitmapWidth], i) + binary.LittleEndian.PutUint64(data[slot:], 0) +} + +func (w *RowWriter) SetNotNullAt(i int) { + w.slot(i) // bounds check + clearBit(w.buf.GetData()[w.base:w.base+w.bitmapWidth], i) +} + +func (w *RowWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *RowWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *RowWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *RowWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *RowWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *RowWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *RowWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *RowWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *RowWriter) WriteTimestamp(i int, t time.Time) { Review Comment: Use a checked timestamp-to-microseconds conversion time.Time.UnixMicro has undefined results when the timestamp cannot be represented as int64 microseconds. This writer currently succeeds with a corrupted value for sufficiently large years. Make timestamp writes return an error and use checked seconds/nanoseconds arithmetic in both RowWriter and ArrayWriter. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil Review Comment: Align temporal field nullability with Java Go infers Date and Timestamp value carriers as non-nullable, whereas Java infers LocalDate, Timestamp, and Instant fields as nullable. Their schema bytes therefore differ, while the type-only hash does not detect the difference; a Java null then becomes a zero Date or year-1 Time in Go. Align the inferred field metadata and require pointer carriers when null must be preserved. ########## go/fory/row/writer.go: ########## @@ -0,0 +1,381 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "encoding/binary" + "fmt" + "math" + "time" + + fory "github.com/apache/fory/go/fory" +) + +const maxArrayDataBytes = math.MaxInt32 - 15 + +// RowWriter writes one row into a ByteBuffer: null bitmap, fixed 8-byte +// field slots, then the variable data region. All slot and bitmap +// offsets are relative to the row base, so a nested RowWriter can share +// the parent's buffer and produce a self-contained row. +// +// Reset rebases the writer at the buffer's current writer index and must +// be called before writing each row. Writers are not goroutine-safe. +type RowWriter struct { + schema *Schema + buf *fory.ByteBuffer + base int + numFields int + bitmapWidth int + fixedSize int +} + +func NewRowWriter(schema *Schema) *RowWriter { + return NewRowWriterWithBuffer(schema, fory.NewByteBuffer(nil)) +} + +// NewRowWriterWithBuffer shares an existing buffer, e.g. to write a +// nested struct row into its parent's variable data region. +func NewRowWriterWithBuffer(schema *Schema, buf *fory.ByteBuffer) *RowWriter { + n := schema.NumFields() + bitmapWidth := bitmapWidthInBytes(n) + return &RowWriter{ + schema: schema, + buf: buf, + numFields: n, + bitmapWidth: bitmapWidth, + fixedSize: bitmapWidth + n*8, + } +} + +func (w *RowWriter) Schema() *Schema { return w.schema } +func (w *RowWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and zeroes the +// whole fixed region. The buffer reuses dirty capacity after shrinking, +// so zeroing here is what makes null slots and padding deterministic; +// it also lets narrow fixed-width writes store just the value. +func (w *RowWriter) Reset() { + w.base = w.buf.WriterIndex() + w.buf.Reserve(w.fixedSize) + data := w.buf.GetData() + clear(data[w.base : w.base+w.fixedSize]) + w.buf.SetWriterIndex(w.base + w.fixedSize) +} + +// Size returns the bytes written for this row so far. +func (w *RowWriter) Size() int { return w.buf.WriterIndex() - w.base } + +// ToBytes returns a view of the row bytes; it stays valid only until the +// buffer is written to or reset again. +func (w *RowWriter) ToBytes() []byte { + return w.buf.GetByteSlice(w.base, w.buf.WriterIndex()) +} + +func (w *RowWriter) slot(i int) int { + if uint(i) >= uint(w.numFields) { + panic(fmt.Sprintf("row: field index %d out of range [0, %d)", i, w.numFields)) + } + return w.base + w.bitmapWidth + i*8 +} + +// SetNullAt marks field i null and zeroes its slot. +func (w *RowWriter) SetNullAt(i int) { + slot := w.slot(i) + data := w.buf.GetData() + setBit(data[w.base:w.base+w.bitmapWidth], i) + binary.LittleEndian.PutUint64(data[slot:], 0) +} + +func (w *RowWriter) SetNotNullAt(i int) { + w.slot(i) // bounds check + clearBit(w.buf.GetData()[w.base:w.base+w.bitmapWidth], i) +} + +func (w *RowWriter) WriteBool(i int, v bool) { + slot := w.slot(i) + if v { + w.buf.GetData()[slot] = 1 + } else { + w.buf.GetData()[slot] = 0 + } +} + +func (w *RowWriter) WriteInt8(i int, v int8) { + w.buf.GetData()[w.slot(i)] = byte(v) +} + +func (w *RowWriter) WriteInt16(i int, v int16) { + binary.LittleEndian.PutUint16(w.buf.GetData()[w.slot(i):], uint16(v)) +} + +func (w *RowWriter) WriteInt32(i int, v int32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], uint32(v)) +} + +func (w *RowWriter) WriteInt64(i int, v int64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(v)) +} + +func (w *RowWriter) WriteFloat32(i int, v float32) { + binary.LittleEndian.PutUint32(w.buf.GetData()[w.slot(i):], math.Float32bits(v)) +} + +func (w *RowWriter) WriteFloat64(i int, v float64) { + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], math.Float64bits(v)) +} + +func (w *RowWriter) WriteDate(i int, d fory.Date) error { + days, err := fory.DateToEpochDay(d) + if err != nil { + return err + } + if days < math.MinInt32 || days > math.MaxInt32 { + return fmt.Errorf("row: date %v out of date32 range", d) + } + w.WriteInt32(i, int32(days)) + return nil +} + +func (w *RowWriter) WriteTimestamp(i int, t time.Time) { + w.WriteInt64(i, t.UnixMicro()) +} + +func (w *RowWriter) WriteDuration(i int, d time.Duration) { + w.WriteInt64(i, d.Microseconds()) +} + +func (w *RowWriter) WriteString(i int, s string) error { + start := appendStringRegion(w.buf, s) + return w.SetOffsetAndSize(i, start, len(s)) +} + +func (w *RowWriter) WriteBytes(i int, b []byte) error { + start := appendBytesRegion(w.buf, b) + return w.SetOffsetAndSize(i, start, len(b)) +} + +// SetOffsetAndSize patches field i's slot with the row-relative offset +// and byte size of a value already appended to the variable data region. +// Use it after writing a nested struct, array, or map at absStart. +// Offsets and sizes beyond 32 bits are rejected: the wire format packs +// both into one slot, and truncating would corrupt the row silently. +func (w *RowWriter) SetOffsetAndSize(i, absStart, size int) error { + rel := absStart - w.base + if uint64(rel) > math.MaxUint32 || uint64(size) > math.MaxUint32 { + return fmt.Errorf("row: value at offset %d with %d bytes exceeds the 32-bit wire format limit", rel, size) + } + binary.LittleEndian.PutUint64(w.buf.GetData()[w.slot(i):], uint64(rel)<<32|uint64(size)) + return nil +} + +// ArrayWriter writes one array: an 8-byte element count, a null bitmap, +// then elements at their natural width (variable-width elements use +// 8-byte offset+size slots). Offsets are relative to the array base. +type ArrayWriter struct { + elem Field + buf *fory.ByteBuffer + base int + elemSize int + numElements int + headerBytes int +} + +func NewArrayWriter(elem Field, buf *fory.ByteBuffer) *ArrayWriter { + elemSize := elem.Type.ByteWidth() + if elemSize < 0 { + elemSize = 8 + } + return &ArrayWriter{elem: elem, buf: buf, elemSize: elemSize} +} + +func (w *ArrayWriter) Buffer() *fory.ByteBuffer { return w.buf } + +// Reset rebases the writer at the current writer index and writes the +// array header plus a zeroed element region for numElements elements. +func (w *ArrayWriter) Reset(numElements int) error { + if numElements < 0 { + return fmt.Errorf("row: negative array length %d", numElements) + } + dataBytes := int64(numElements) * int64(w.elemSize) Review Comment: Check the array limit before multiplying With an 8-byte element, Reset(math.MaxInt) overflows dataBytes to a negative value, bypasses the maximum-size check, and later panics while computing or reserving the region. Compare numElements against maxArrayDataBytes/elemSize before multiplying and test that extreme input returns an error. ########## go/fory/row/schema_bytes.go: ########## @@ -0,0 +1,351 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "math" + + fory "github.com/apache/fory/go/fory" + "github.com/apache/fory/go/fory/meta" +) + +// Schema wire format, byte-compatible with Java SchemaEncoder: +// +// | version (1 byte) | num_fields (var uint32 small7) | fields... | +// +// Field: | header (1 byte) | name size (varint, only if large) | name +// bytes | type info |. Header bits 0-1 hold the index into +// fieldNameEncodings, bits 2-5 hold nameSize-1 (15 means a varint with +// nameSize-15 follows), bit 6 is the nullable flag. Type info is the +// type id byte, plus precision+scale for DECIMAL, a recursive element +// field for LIST, recursive key and value fields for MAP, and a field +// count plus recursive fields for STRUCT. +const ( + schemaVersion = 1 + fieldNameSizeThreshold = 15 + // Caps readField/readType recursion. + maxSchemaNestingDepth = 64 +) + +// Header bits 0-1 store the INDEX into this table (matching Java +// SchemaEncoder.FIELD_NAME_ENCODINGS order), not the meta.Encoding +// wire values, which differ. +var fieldNameEncodings = [3]meta.Encoding{ + meta.UTF_8, + meta.ALL_TO_LOWER_SPECIAL, + meta.LOWER_UPPER_DIGIT_SPECIAL, +} + +// Stateless after construction, safe for concurrent use. +var ( + fieldNameEncoder = meta.NewEncoder('$', '_') + fieldNameDecoder = meta.NewDecoder('$', '_') +) + +func encodingIndexOf(encoding meta.Encoding) int { + for i, e := range fieldNameEncodings { + if e == encoding { + return i + } + } + return -1 +} + +// SchemaToBytes serializes a schema to the cross-language wire format. +func SchemaToBytes(s *Schema) ([]byte, error) { + buf := fory.NewByteBuffer(nil) + if err := SchemaToBuffer(s, buf); err != nil { + return nil, err + } + return buf.GetByteSlice(0, buf.WriterIndex()), nil +} + +// SchemaToBuffer serializes a schema into an existing buffer. +func SchemaToBuffer(s *Schema, buf *fory.ByteBuffer) error { + buf.WriteByte_(schemaVersion) + buf.WriteVarUint32Small7(uint32(s.NumFields())) + for _, f := range s.Fields() { + if err := writeSchemaField(buf, f); err != nil { + return err + } + } + return nil +} + +func writeSchemaField(buf *fory.ByteBuffer, f Field) error { + if f.Name == "" { + return fmt.Errorf("row: schema field with empty name") + } + encoding := fieldNameEncoder.ComputeEncodingWith(f.Name, fieldNameEncodings[:]) + metaString, err := fieldNameEncoder.EncodeWithEncoding(f.Name, encoding) + if err != nil { + return fmt.Errorf("row: encoding field name %q: %w", f.Name, err) + } + nameBytes := metaString.GetEncodedBytes() + nameSize := len(nameBytes) + encodingIndex := encodingIndexOf(metaString.GetEncoding()) + if encodingIndex < 0 { + return fmt.Errorf("row: field name %q got unsupported encoding %d", f.Name, metaString.GetEncoding()) + } + + header := encodingIndex & 0x03 + bigSize := nameSize > fieldNameSizeThreshold + if bigSize { + header |= fieldNameSizeThreshold << 2 + } else { + header |= (nameSize - 1) << 2 + } + if f.Nullable { + header |= 0x40 + } + buf.WriteByte_(byte(header)) + if bigSize { + buf.WriteVarUint32Small7(uint32(nameSize - fieldNameSizeThreshold)) + } + buf.WriteBinary(nameBytes) + return writeSchemaType(buf, f.Type) +} + +func writeSchemaType(buf *fory.ByteBuffer, dataType DataType) error { Review Comment: Validate schema types, depth, and cycles before writing The writer can emit a 65-level schema that its own reader rejects, and a self-referential ListType or StructType recurses until a fatal stack overflow. Dynamic type handling is also incomplete: *DecimalType writes only the type ID, out-of-range precision/scale is silently truncated to byte, and an external composite DataType can omit child metadata. Perform an exhaustive validation pass before writing any bytes. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { Review Comment: Reject pointers to already-nilable carriers For *[]T, *map[K]V, and *[]byte, a nil outer pointer and a non-nil pointer to a nil container both produce the same row null bit. Decode always returns a nil outer pointer, so valid Go values cannot round-trip. Reject these double-nilable shapes unless the format gains a separate representation for both states. ########## go/fory/row/infer.go: ########## @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + fory "github.com/apache/fory/go/fory" +) + +var ( + goDateType = reflect.TypeOf(fory.Date{}) + goTimeType = reflect.TypeOf(time.Time{}) + goDurationType = reflect.TypeOf(time.Duration(0)) +) + +// InferSchema infers the row schema for a struct type or pointer to +// struct, including every exported field not tagged `fory:"ignore"`. +// +// Fields are sorted by their lowerCamel name and named by its +// snake_case form (UserName -> user_name), matching Java's schema +// inference so both languages derive identical schemas. +// +// Type mapping: +// - bool, int8/16/32/64, float32/64: same-width row types; int maps to int64 +// - string, []byte: string and binary, nullable +// - slices, maps, nested structs: list, map, and struct, nullable +// - *T: the row type of T, nullable +// - fory.Date, time.Time, time.Duration: date32, timestamp, duration +// +// Unsigned integers, fixed-size arrays, nested pointers, and pointer +// map keys are unsupported and return an error. +func InferSchema(t reflect.Type) (*Schema, error) { + layout, err := inferStructLayout(t, nil) + if err != nil { + return nil, err + } + return layout.schema, nil +} + +// structLayout maps schema ordinals back to Go struct field indexes. +type structLayout struct { + schema *Schema + indexes []int // schema ordinal -> reflect struct field index +} + +func inferStructLayout(t reflect.Type, path []reflect.Type) (*structLayout, error) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct || t == goDateType || t == goTimeType { + return nil, fmt.Errorf("row: schema inference expects a struct type, got %v", t) + } + for _, seen := range path { + if seen == t { + return nil, fmt.Errorf("row: circular reference through type %v", t) + } + } + path = append(path, t) + + type member struct { + lowerCamel string + goIndex int + fieldType reflect.Type + } + var members []member + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue + } + ignored, err := hasIgnoreTag(f.Tag.Get("fory")) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, f.Name, t) + } + if ignored { + continue + } + members = append(members, member{lowerFirst(f.Name), i, f.Type}) + } + // Java sorts by the lowerCamel member name before converting names + // to snake_case; matching that order is required for schema and + // row-layout compatibility. + sort.Slice(members, func(a, b int) bool { return members[a].lowerCamel < members[b].lowerCamel }) + + layout := &structLayout{indexes: make([]int, 0, len(members))} + fields := make([]Field, 0, len(members)) + for _, m := range members { + f, err := inferField(lowerCamelToLowerUnderscore(m.lowerCamel), m.fieldType, path) + if err != nil { + return nil, fmt.Errorf("%w (field %s of %v)", err, t.Field(m.goIndex).Name, t) + } + fields = append(fields, f) + layout.indexes = append(layout.indexes, m.goIndex) + } + layout.schema = NewSchema(fields) + return layout, nil +} + +func inferField(name string, t reflect.Type, path []reflect.Type) (Field, error) { + if t.Kind() == reflect.Ptr { + if t.Elem().Kind() == reflect.Ptr { + return Field{}, fmt.Errorf("row: nested pointer type %v is unsupported", t) + } + inner, err := inferField(name, t.Elem(), path) + if err != nil { + return Field{}, err + } + inner.Nullable = true + return inner, nil + } + switch t { + case goDateType: + return Field{Name: name, Type: Date32Type{}}, nil + case goTimeType: + return Field{Name: name, Type: TimestampType{}}, nil + case goDurationType: + return Field{Name: name, Type: DurationType{}}, nil + } + switch t.Kind() { + case reflect.Bool: + return Field{Name: name, Type: BoolType{}}, nil + case reflect.Int8: + return Field{Name: name, Type: Int8Type{}}, nil + case reflect.Int16: + return Field{Name: name, Type: Int16Type{}}, nil + case reflect.Int32: + return Field{Name: name, Type: Int32Type{}}, nil + case reflect.Int64, reflect.Int: + // `int` maps to int64 so the width never depends on the platform. + return Field{Name: name, Type: Int64Type{}}, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return Field{}, fmt.Errorf("row: unsigned type %v is unsupported, use a signed type", t) + case reflect.Float32: + return Field{Name: name, Type: Float32Type{}}, nil + case reflect.Float64: + return Field{Name: name, Type: Float64Type{}}, nil + case reflect.String: + return Field{Name: name, Type: StringType{}, Nullable: true}, nil + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return Field{Name: name, Type: BinaryType{}, Nullable: true}, nil + } + elem, err := inferField(listItemName, t.Elem(), path) + if err != nil { + return Field{}, err + } + return Field{Name: name, Type: &ListType{Elem: elem}, Nullable: true}, nil + case reflect.Map: + if t.Key().Kind() == reflect.Ptr { Review Comment: Require map keys to preserve Go equality Rejecting only a direct pointer key is insufficient. A comparable struct key can contain ignored or unexported equality state, causing distinct keys to encode identically and one entry to overwrite the other on decode; pointer identity nested inside a struct is also not preserved. Recursively restrict keys to shapes whose encoded fields fully determine Go ==. ########## go/fory/row/datatype.go: ########## @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package row + +import ( + "fmt" + "strings" + + fory "github.com/apache/fory/go/fory" +) + +// DataType describes the row-format type of a field. +// +// Type ids are the cross-language Fory type ids shared with the Java, +// C++, and Python row-format implementations; they appear verbatim in +// the serialized schema wire format. +type DataType interface { + TypeID() fory.TypeId + // ByteWidth returns the fixed storage width in bytes, or -1 for + // variable-width types stored through an offset+size slot. + ByteWidth() int + String() string +} + +// Cross-language child field names for matching Java DataTypes. +const ( + listItemName = "item" + mapKeyName = "key" + mapValueName = "value" +) + +type BoolType struct{} + +func (BoolType) TypeID() fory.TypeId { return fory.BOOL } +func (BoolType) ByteWidth() int { return 1 } +func (BoolType) String() string { return "bool" } + +type Int8Type struct{} + +func (Int8Type) TypeID() fory.TypeId { return fory.INT8 } +func (Int8Type) ByteWidth() int { return 1 } +func (Int8Type) String() string { return "int8" } + +type Int16Type struct{} + +func (Int16Type) TypeID() fory.TypeId { return fory.INT16 } +func (Int16Type) ByteWidth() int { return 2 } +func (Int16Type) String() string { return "int16" } + +type Int32Type struct{} + +func (Int32Type) TypeID() fory.TypeId { return fory.INT32 } +func (Int32Type) ByteWidth() int { return 4 } +func (Int32Type) String() string { return "int32" } + +type Int64Type struct{} + +func (Int64Type) TypeID() fory.TypeId { return fory.INT64 } +func (Int64Type) ByteWidth() int { return 8 } +func (Int64Type) String() string { return "int64" } + +// Float16Type exists so schemas received from other languages parse, +// the Go writer, reader, and encoder do not support it yet. +type Float16Type struct{} + +func (Float16Type) TypeID() fory.TypeId { return fory.FLOAT16 } +func (Float16Type) ByteWidth() int { return 2 } +func (Float16Type) String() string { return "float16" } + +type Float32Type struct{} + +func (Float32Type) TypeID() fory.TypeId { return fory.FLOAT32 } +func (Float32Type) ByteWidth() int { return 4 } +func (Float32Type) String() string { return "float32" } + +type Float64Type struct{} + +func (Float64Type) TypeID() fory.TypeId { return fory.FLOAT64 } +func (Float64Type) ByteWidth() int { return 8 } +func (Float64Type) String() string { return "float64" } + +// StringType values are UTF-8 bytes in the variable data region, +// stored as (offset + size) in the fixed slot region. +type StringType struct{} + +func (StringType) TypeID() fory.TypeId { return fory.STRING } +func (StringType) ByteWidth() int { return -1 } +func (StringType) String() string { return "string" } + +type BinaryType struct{} + +func (BinaryType) TypeID() fory.TypeId { return fory.BINARY } +func (BinaryType) ByteWidth() int { return -1 } +func (BinaryType) String() string { return "binary" } + +// Date32Type values are days since the Unix epoch (int32). +type Date32Type struct{} + +func (Date32Type) TypeID() fory.TypeId { return fory.DATE } +func (Date32Type) ByteWidth() int { return 4 } +func (Date32Type) String() string { return "date32" } + +// TimestampType values are microseconds since the Unix epoch (int64). +type TimestampType struct{} + +func (TimestampType) TypeID() fory.TypeId { return fory.TIMESTAMP } +func (TimestampType) ByteWidth() int { return 8 } +func (TimestampType) String() string { return "timestamp" } + +// DurationType values are microseconds (int64). +type DurationType struct{} + +func (DurationType) TypeID() fory.TypeId { return fory.DURATION } +func (DurationType) ByteWidth() int { return 8 } +func (DurationType) String() string { return "duration" } + +// DecimalType exists so schemas received from other languages parse; the +// Go writer, reader, and encoder do not support it yet. Use it as a +// value, not a pointer, so schema equality compares precision and scale. +type DecimalType struct { + Precision int + Scale int +} + +func (DecimalType) TypeID() fory.TypeId { return fory.DECIMAL } +func (DecimalType) ByteWidth() int { return -1 } +func (t DecimalType) String() string { + return fmt.Sprintf("decimal(%d, %d)", t.Precision, t.Scale) +} + +// ListType is a variable-length sequence of Elem values. +type ListType struct { + Elem Field +} + +func (*ListType) TypeID() fory.TypeId { return fory.LIST } +func (*ListType) ByteWidth() int { return -1 } +func (t *ListType) String() string { return "list<" + t.Elem.Type.String() + ">" } + +// MapType stores keys and values as two adjacent arrays. +type MapType struct { + Key Field + Value Field +} + +func (*MapType) TypeID() fory.TypeId { return fory.MAP } +func (*MapType) ByteWidth() int { return -1 } +func (t *MapType) String() string { + return "map<" + t.Key.Type.String() + ", " + t.Value.Type.String() + ">" +} + +// StructType is a nested row with its own bitmap, slots, and variable +// data region. +type StructType struct { + Fields []Field +} + +func (*StructType) TypeID() fory.TypeId { return fory.STRUCT } +func (*StructType) ByteWidth() int { return -1 } +func (t *StructType) String() string { + var sb strings.Builder + sb.WriteString("struct<") + for i, f := range t.Fields { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(f.Name) + sb.WriteString(": ") + sb.WriteString(f.Type.String()) + } + sb.WriteString(">") + return sb.String() +} + +// Field is a named, optionally nullable slot in a schema or composite type. +type Field struct { + Name string + Type DataType + Nullable bool +} + +func NewField(name string, dataType DataType, nullable bool) Field { + return Field{Name: name, Type: dataType, Nullable: nullable} +} + +func (f Field) Equal(o Field) bool { + return f.Name == o.Name && f.Nullable == o.Nullable && dataTypeEqual(f.Type, o.Type) +} + +// List returns a list type with the cross-language element field name +// "item" and nullable elements. Build ListType directly for +// non-nullable elements. +func List(elem DataType) *ListType { + return &ListType{Elem: Field{Name: listItemName, Type: elem, Nullable: true}} +} + +// Map returns a map type with the cross-language field names +// "key"/"value". Keys are always non-nullable. +func Map(key, value DataType) *MapType { + return &MapType{ + Key: Field{Name: mapKeyName, Type: key, Nullable: false}, + Value: Field{Name: mapValueName, Type: value, Nullable: true}, + } +} + +func Struct(fields []Field) *StructType { + return &StructType{Fields: fields} +} + +func dataTypeEqual(a, b DataType) bool { + switch at := a.(type) { + case *ListType: + bt, ok := b.(*ListType) + return ok && at.Elem.Equal(bt.Elem) + case *MapType: + bt, ok := b.(*MapType) + return ok && at.Key.Equal(bt.Key) && at.Value.Equal(bt.Value) + case *StructType: + bt, ok := b.(*StructType) + if !ok || len(at.Fields) != len(bt.Fields) { + return false + } + for i := range at.Fields { + if !at.Fields[i].Equal(bt.Fields[i]) { + return false + } + } + return true + default: + // Primitive types are empty structs and DecimalType is a + // comparable value, so interface equality is exact. + return a == b + } +} + +// Schema describes the fields of a top-level row. +type Schema struct { + fields []Field + byName map[string]int +} + +// NewSchema builds a schema from fields in their schema-declared order, +// which fixes the field slot layout. For duplicate names, FieldIndex +// resolves to the first occurrence. +func NewSchema(fields []Field) *Schema { + byName := make(map[string]int, len(fields)) + for i, f := range fields { + if _, ok := byName[f.Name]; !ok { + byName[f.Name] = i + } + } + return &Schema{fields: fields, byName: byName} Review Comment: Copy schema fields before caching their indexes NewSchema stores the caller's slice after building byName. Mutating the original fields slice can therefore make Field(i) report a new name while FieldIndex still resolves the old one. Defensively copy the input, including nested schema state that participates in cached encoder/hash behavior. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
