rambleraptor commented on code in PR #1212: URL: https://github.com/apache/iceberg-go/pull/1212#discussion_r3462953889
########## expr_json.go: ########## @@ -0,0 +1,594 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iceberg + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/google/uuid" +) + +// JSON for boolean expressions, used as the "filter" of a REST scan-planning +// request. Mirrors Java's ExpressionParser. + +// exprKeyTransform is the "type" value identifying a transform term. +const exprKeyTransform = "transform" + +// opToJSON maps an Operation to its wire string (OpLTEQ -> "lt-eq"). OpTrue and +// OpFalse are handled separately: they serialize as bare JSON booleans. +var opToJSON = map[Operation]string{ + OpIsNull: "is-null", + OpNotNull: "not-null", + OpIsNan: "is-nan", + OpNotNan: "not-nan", + OpLT: "lt", + OpLTEQ: "lt-eq", + OpGT: "gt", + OpGTEQ: "gt-eq", + OpEQ: "eq", + OpNEQ: "not-eq", + OpStartsWith: "starts-with", + OpNotStartsWith: "not-starts-with", + OpIn: "in", + OpNotIn: "not-in", + OpNot: "not", + OpAnd: "and", + OpOr: "or", +} + +var jsonToOp = func() map[string]Operation { + m := make(map[string]Operation, len(opToJSON)) + for op, s := range opToJSON { + m[s] = op + } + + return m +}() + +// exprNode is the wire form of an expression node. omitempty leaves only the +// keys relevant to a given node; field order matches Java's output. +type exprNode struct { + Type string `json:"type"` + Child json.RawMessage `json:"child,omitempty"` + Left json.RawMessage `json:"left,omitempty"` + Right json.RawMessage `json:"right,omitempty"` + Term json.RawMessage `json:"term,omitempty"` + Value json.RawMessage `json:"value,omitempty"` + Values []json.RawMessage `json:"values,omitempty"` +} + +// transformNode is the wire form of a transform term, e.g. +// {"type":"transform","transform":"bucket[16]","term":"id"}. +type transformNode struct { + Type string `json:"type"` + Transform string `json:"transform"` + Term string `json:"term"` +} + +// ParseExpr parses an expression from its REST JSON form (a request "filter" or +// a task's residual filter). +// +// With a schema, literals take the referenced field's type (e.g. "2022-08-14" +// on a date column becomes a DateLiteral). Without one they fall back to the +// base JSON kind: Int64Literal, Float64Literal, StringLiteral, or BoolLiteral. +func ParseExpr(data []byte, schema *Schema) (BooleanExpression, error) { + return decodeExpr(json.RawMessage(data), schema) +} + +// MarshalJSON emits the REST form, so an expression can be used as a "filter" +// field directly. Tag such fields omitempty to drop a nil one. +func (AlwaysTrue) MarshalJSON() ([]byte, error) { return []byte("true"), nil } +func (AlwaysFalse) MarshalJSON() ([]byte, error) { return []byte("false"), nil } + +func (n NotExpr) MarshalJSON() ([]byte, error) { + child, err := json.Marshal(n.child) + if err != nil { + return nil, err + } + + return json.Marshal(exprNode{Type: opToJSON[OpNot], Child: child}) +} + +func (a AndExpr) MarshalJSON() ([]byte, error) { + left, right, err := marshalChildren(a.left, a.right) + if err != nil { + return nil, err + } + + return json.Marshal(exprNode{Type: opToJSON[OpAnd], Left: left, Right: right}) +} + +func (o OrExpr) MarshalJSON() ([]byte, error) { + left, right, err := marshalChildren(o.left, o.right) + if err != nil { + return nil, err + } + + return json.Marshal(exprNode{Type: opToJSON[OpOr], Left: left, Right: right}) +} + +func marshalChildren(left, right BooleanExpression) (json.RawMessage, json.RawMessage, error) { + l, err := json.Marshal(left) + if err != nil { + return nil, nil, err + } + r, err := json.Marshal(right) + if err != nil { + return nil, nil, err + } + + return l, r, nil +} + +// Bound predicates need their own MarshalJSON too; without it json.Marshal would +// fall through to {} since their fields are unexported. +func (up *unboundUnaryPredicate) MarshalJSON() ([]byte, error) { + return marshalUnaryPredicate(up.op, up.term) +} + +func (bp *boundUnaryPredicate[T]) MarshalJSON() ([]byte, error) { + return marshalUnaryPredicate(bp.op, bp.term) +} + +func (ul *unboundLiteralPredicate) MarshalJSON() ([]byte, error) { + return marshalLiteralPredicate(ul.op, ul.term, ul.lit) +} + +func (blp *boundLiteralPredicate[T]) MarshalJSON() ([]byte, error) { + return marshalLiteralPredicate(blp.op, blp.term, blp.lit) +} + +func (usp *unboundSetPredicate) MarshalJSON() ([]byte, error) { + return marshalSetPredicate(usp.op, usp.term, usp.lits.Members()) +} + +func (bsp *boundSetPredicate[T]) MarshalJSON() ([]byte, error) { + return marshalSetPredicate(bsp.op, bsp.term, bsp.lits.Members()) +} + +// predicateType returns the wire "type" string for a predicate operation. +func predicateType(op Operation) (string, error) { + s, ok := opToJSON[op] + if !ok { + return "", fmt.Errorf("%w: cannot serialize expression with operation %s", ErrInvalidArgument, op) + } + + return s, nil +} + +func marshalUnaryPredicate(op Operation, term Term) ([]byte, error) { + js, err := predicateType(op) + if err != nil { + return nil, err + } + t, err := json.Marshal(term) + if err != nil { + return nil, err + } + + return json.Marshal(exprNode{Type: js, Term: t}) +} + +func marshalLiteralPredicate(op Operation, term Term, lit Literal) ([]byte, error) { + js, err := predicateType(op) + if err != nil { + return nil, err + } + t, err := json.Marshal(term) + if err != nil { + return nil, err + } + v, err := json.Marshal(literalValue{lit, termType(term)}) + if err != nil { + return nil, err + } + + return json.Marshal(exprNode{Type: js, Term: t, Value: v}) +} + +// termType is the field type of a bound term, or nil for an unbound reference. +func termType(term Term) Type { + if bt, ok := term.(BoundTerm); ok { + return bt.Ref().Field().Type + } + + return nil +} + +func marshalSetPredicate(op Operation, term Term, lits []Literal) ([]byte, error) { + js, err := predicateType(op) + if err != nil { + return nil, err + } + t, err := json.Marshal(term) + if err != nil { + return nil, err + } + + typ := termType(term) + values := make([]json.RawMessage, 0, len(lits)) + for _, l := range lits { + v, err := json.Marshal(literalValue{l, typ}) + if err != nil { + return nil, err + } + values = append(values, v) + } + // A set has no order; sort the encoded values for deterministic output. + sort.Slice(values, func(i, j int) bool { + return bytes.Compare(values[i], values[j]) < 0 + }) + + return json.Marshal(exprNode{Type: js, Term: t, Values: values}) +} + +func (r Reference) MarshalJSON() ([]byte, error) { return json.Marshal(string(r)) } + +func (b *boundRef[T]) MarshalJSON() ([]byte, error) { return json.Marshal(b.field.Name) } + +func (b *BoundTransform) MarshalJSON() ([]byte, error) { + return json.Marshal(transformNode{ + Type: exprKeyTransform, + Transform: b.transform.String(), + Term: b.term.Ref().Field().Name, + }) +} + +// Each literal writes its REST form (Java's SingleValueParser). +func (l BoolLiteral) MarshalJSON() ([]byte, error) { return json.Marshal(bool(l)) } +func (l Int32Literal) MarshalJSON() ([]byte, error) { return json.Marshal(int32(l)) } +func (l Int64Literal) MarshalJSON() ([]byte, error) { return json.Marshal(int64(l)) } +func (l StringLiteral) MarshalJSON() ([]byte, error) { return json.Marshal(string(l)) } + +func (l Float32Literal) MarshalJSON() ([]byte, error) { + if f := float64(l); math.IsInf(f, 0) || math.IsNaN(f) { + return nil, fmt.Errorf("%w: cannot serialize non-finite float %v", ErrInvalidArgument, f) + } + + return json.Marshal(float32(l)) +} + +func (l Float64Literal) MarshalJSON() ([]byte, error) { + if f := float64(l); math.IsInf(f, 0) || math.IsNaN(f) { + return nil, fmt.Errorf("%w: cannot serialize non-finite float %v", ErrInvalidArgument, f) + } + + return json.Marshal(float64(l)) +} + +func (l DateLiteral) MarshalJSON() ([]byte, error) { + return json.Marshal(Date(l).ToTime().Format("2006-01-02")) +} + +func (l TimeLiteral) MarshalJSON() ([]byte, error) { + // "9"s trim trailing fractional zeros (and the point when zero), as Java does. + return json.Marshal(time.UnixMicro(int64(l)).UTC().Format("15:04:05.999999")) +} + +// A bare timestamp literal has no field type, so it can't pick a wire form; +// serialize through a predicate (see literalValue) instead. +func (TimestampLiteral) MarshalJSON() ([]byte, error) { + return nil, fmt.Errorf("%w: serialize a timestamp literal through a predicate so the field type is known", ErrInvalidArgument) +} + +func (TimestampNsLiteral) MarshalJSON() ([]byte, error) { + return nil, fmt.Errorf("%w: serialize a timestamp literal through a predicate so the field type is known", ErrInvalidArgument) +} + +func (l UUIDLiteral) MarshalJSON() ([]byte, error) { + return json.Marshal(uuid.UUID(l).String()) +} + +func (l FixedLiteral) MarshalJSON() ([]byte, error) { + return json.Marshal(strings.ToUpper(hex.EncodeToString([]byte(l)))) +} + +func (l BinaryLiteral) MarshalJSON() ([]byte, error) { + return json.Marshal(strings.ToUpper(hex.EncodeToString([]byte(l)))) +} + +func (l DecimalLiteral) MarshalJSON() ([]byte, error) { + return json.Marshal(Decimal(l).String()) +} + +// literalValue pairs a literal with its field type, like typeIFace does for +// types. timestamp and timestamptz share one literal type, so the field type is +// the only thing that decides the +00:00 offset on the wire. +type literalValue struct { + Literal + typ Type +} + +func (v literalValue) MarshalJSON() ([]byte, error) { + switch l := v.Literal.(type) { + case TimestampLiteral: + return marshalTimestamp(Timestamp(l).ToTime(), v.typ) + case TimestampNsLiteral: + return marshalTimestamp(TimestampNano(l).ToTime(), v.typ) + default: + return json.Marshal(v.Literal) + } +} + +func (v *literalValue) UnmarshalJSON(b []byte) error { + base, err := asObject(b) + if err != nil { + return err + } + lit, err := convertValue(base, v.typ) + if err != nil { + return err + } + v.Literal = lit + + return nil +} + +// marshalTimestamp emits the REST form: with the UTC offset for timestamptz, +// without it for timestamp. +func marshalTimestamp(tm time.Time, typ Type) ([]byte, error) { + switch typ.(type) { + case TimestampTzType, TimestampTzNsType: + // "-07:00" prints +00:00 for a UTC instant, matching Java. + return json.Marshal(tm.Format("2006-01-02T15:04:05.999999-07:00")) Review Comment: Let's just fix it while we're here. If you're up for the review, I'd rather try to get it right here, rather than have it linger for an undetermined amount of time. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
