pitrou commented on code in PR #13792: URL: https://github.com/apache/arrow/pull/13792#discussion_r937929132
########## go/arrow/decimal256/decimal256.go: ########## @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal256 + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/apache/arrow/go/v10/arrow/decimal128" + "github.com/apache/arrow/go/v10/arrow/internal/debug" +) + +const ( + MaxPrecision = 76 + MaxScale = 76 +) + +type Num struct { + // arr[0] is the lowest bits, arr[3] is the highest bits + arr [4]uint64 +} + +// New returns a new signed 256-bit integer value where x1 contains +// the highest bits with the rest of the values in order down to the +// lowest bits +// +// ie: New(1, 2, 3, 4) returns with the elements in little-endian order +// {4, 3, 2, 1} but each value is still represented as the native endianness +func New(x1, x2, x3, x4 uint64) Num { + return Num{[4]uint64{x4, x3, x2, x1}} +} + +func (n Num) Array() [4]uint64 { return n.arr } + +func (n Num) LowBits() uint64 { return n.arr[0] } + +func FromDecimal128(n decimal128.Num) Num { + var topBits uint64 + if n.Sign() < 0 { + topBits = math.MaxUint64 + } + return New(topBits, topBits, uint64(n.HighBits()), n.LowBits()) +} + +func FromU64(v uint64) Num { + return Num{[4]uint64{v, 0, 0, 0}} +} + +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, 0, 0, uint64(v)) + case v < 0: + return New(math.MaxUint64, math.MaxUint64, math.MaxUint64, uint64(v)) + default: + return Num{} + } +} + +func (n Num) Negate() Num { + var carry uint64 = 1 + for i := range n.arr { + n.arr[i] = ^n.arr[i] + carry + if n.arr[i] != 0 { + carry = 0 + } + } + return n +} + +func FromFloat32(v float32, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(float64(v), 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat32(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat32(v, prec, scale) +} + +func fromPositiveFloat32(v float32, prec, scale int32) (Num, error) { + var pscale float32 + if scale >= -76 && scale <= 76 { + pscale = float32PowersOfTen[scale+76] + } else { + pscale = float32(math.Pow10(int(scale))) + } + + v *= pscale + v = float32(math.RoundToEven(float64(v))) + maxabs := float32PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float32 + arr[3] = float32(math.Floor(math.Ldexp(float64(v), -192))) + v -= float32(math.Ldexp(float64(arr[3]), 192)) + arr[2] = float32(math.Floor(math.Ldexp(float64(v), -128))) + v -= float32(math.Ldexp(float64(arr[2]), 128)) + arr[1] = float32(math.Floor(math.Ldexp(float64(v), -64))) + v -= float32(math.Ldexp(float64(arr[1]), 64)) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func FromFloat64(v float64, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(v, 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat64(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat64(v, prec, scale) +} + +func fromPositiveFloat64(v float64, prec, scale int32) (Num, error) { + var pscale float64 + if scale >= -76 && scale <= 76 { + pscale = float64PowersOfTen[scale+76] + } else { + pscale = math.Pow10(int(scale)) + } + + v *= pscale + v = math.RoundToEven(v) + maxabs := float64PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float64 + arr[3] = math.Floor(math.Ldexp(v, -192)) + v -= math.Ldexp(arr[3], 192) + arr[2] = math.Floor(math.Ldexp(v, -128)) + v -= math.Ldexp(arr[2], 128) + arr[1] = math.Floor(math.Ldexp(v, -64)) + v -= math.Ldexp(arr[1], 64) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func (n Num) tofloat32Positive(scale int32) float32 { + const twoTo64 float32 = 1.8446744e+19 Review Comment: Here as well, it might be better to do the calculation in the float64 realm? ########## go/arrow/decimal256/decimal256.go: ########## @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal256 + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/apache/arrow/go/v10/arrow/decimal128" + "github.com/apache/arrow/go/v10/arrow/internal/debug" +) + +const ( + MaxPrecision = 76 + MaxScale = 76 +) + +type Num struct { + // arr[0] is the lowest bits, arr[3] is the highest bits + arr [4]uint64 +} + +// New returns a new signed 256-bit integer value where x1 contains +// the highest bits with the rest of the values in order down to the +// lowest bits +// +// ie: New(1, 2, 3, 4) returns with the elements in little-endian order +// {4, 3, 2, 1} but each value is still represented as the native endianness +func New(x1, x2, x3, x4 uint64) Num { + return Num{[4]uint64{x4, x3, x2, x1}} +} + +func (n Num) Array() [4]uint64 { return n.arr } + +func (n Num) LowBits() uint64 { return n.arr[0] } + +func FromDecimal128(n decimal128.Num) Num { + var topBits uint64 + if n.Sign() < 0 { + topBits = math.MaxUint64 + } + return New(topBits, topBits, uint64(n.HighBits()), n.LowBits()) +} + +func FromU64(v uint64) Num { + return Num{[4]uint64{v, 0, 0, 0}} +} + +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, 0, 0, uint64(v)) + case v < 0: + return New(math.MaxUint64, math.MaxUint64, math.MaxUint64, uint64(v)) + default: + return Num{} + } +} + +func (n Num) Negate() Num { + var carry uint64 = 1 + for i := range n.arr { + n.arr[i] = ^n.arr[i] + carry + if n.arr[i] != 0 { + carry = 0 + } + } + return n +} + +func FromFloat32(v float32, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(float64(v), 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat32(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat32(v, prec, scale) +} + +func fromPositiveFloat32(v float32, prec, scale int32) (Num, error) { + var pscale float32 + if scale >= -76 && scale <= 76 { + pscale = float32PowersOfTen[scale+76] Review Comment: Wouldn't it be better to cast `v` to float64 and then call `fromPositive64`? You would 1) remove a lot of code 2) get better precision (`float32PowersOfTen` contains some zeros and infinities for very small or large scales) ########## go/arrow/decimal256/decimal256.go: ########## @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal256 + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/apache/arrow/go/v10/arrow/decimal128" + "github.com/apache/arrow/go/v10/arrow/internal/debug" +) + +const ( + MaxPrecision = 76 + MaxScale = 76 +) + +type Num struct { + // arr[0] is the lowest bits, arr[3] is the highest bits + arr [4]uint64 +} + +// New returns a new signed 256-bit integer value where x1 contains +// the highest bits with the rest of the values in order down to the +// lowest bits +// +// ie: New(1, 2, 3, 4) returns with the elements in little-endian order +// {4, 3, 2, 1} but each value is still represented as the native endianness +func New(x1, x2, x3, x4 uint64) Num { + return Num{[4]uint64{x4, x3, x2, x1}} +} + +func (n Num) Array() [4]uint64 { return n.arr } + +func (n Num) LowBits() uint64 { return n.arr[0] } + +func FromDecimal128(n decimal128.Num) Num { + var topBits uint64 + if n.Sign() < 0 { + topBits = math.MaxUint64 + } + return New(topBits, topBits, uint64(n.HighBits()), n.LowBits()) +} + +func FromU64(v uint64) Num { + return Num{[4]uint64{v, 0, 0, 0}} +} + +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, 0, 0, uint64(v)) + case v < 0: + return New(math.MaxUint64, math.MaxUint64, math.MaxUint64, uint64(v)) + default: + return Num{} + } +} + +func (n Num) Negate() Num { + var carry uint64 = 1 + for i := range n.arr { + n.arr[i] = ^n.arr[i] + carry + if n.arr[i] != 0 { + carry = 0 + } + } + return n +} + +func FromFloat32(v float32, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(float64(v), 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat32(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat32(v, prec, scale) +} + +func fromPositiveFloat32(v float32, prec, scale int32) (Num, error) { + var pscale float32 + if scale >= -76 && scale <= 76 { + pscale = float32PowersOfTen[scale+76] + } else { + pscale = float32(math.Pow10(int(scale))) + } + + v *= pscale + v = float32(math.RoundToEven(float64(v))) + maxabs := float32PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float32 + arr[3] = float32(math.Floor(math.Ldexp(float64(v), -192))) + v -= float32(math.Ldexp(float64(arr[3]), 192)) + arr[2] = float32(math.Floor(math.Ldexp(float64(v), -128))) + v -= float32(math.Ldexp(float64(arr[2]), 128)) + arr[1] = float32(math.Floor(math.Ldexp(float64(v), -64))) + v -= float32(math.Ldexp(float64(arr[1]), 64)) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func FromFloat64(v float64, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(v, 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat64(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat64(v, prec, scale) +} + +func fromPositiveFloat64(v float64, prec, scale int32) (Num, error) { + var pscale float64 + if scale >= -76 && scale <= 76 { + pscale = float64PowersOfTen[scale+76] + } else { + pscale = math.Pow10(int(scale)) + } + + v *= pscale + v = math.RoundToEven(v) + maxabs := float64PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float64 + arr[3] = math.Floor(math.Ldexp(v, -192)) + v -= math.Ldexp(arr[3], 192) + arr[2] = math.Floor(math.Ldexp(v, -128)) + v -= math.Ldexp(arr[2], 128) + arr[1] = math.Floor(math.Ldexp(v, -64)) + v -= math.Ldexp(arr[1], 64) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func (n Num) tofloat32Positive(scale int32) float32 { + const twoTo64 float32 = 1.8446744e+19 + if n.arr[3] != 0 || n.arr[2] != 0 { + return floatInf Review Comment: The decimal256 might have a large integer value but with a scale that compensates... ########## go/arrow/datatype_fixedwidth.go: ########## @@ -529,6 +529,26 @@ func (Decimal128Type) Layout() DataTypeLayout { return DataTypeLayout{Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(Decimal128SizeBytes)}} } +// Decimal128Type represents a fixed-size 128-bit decimal type. +type Decimal256Type struct { + Precision int32 + Scale int32 +} + +func (*Decimal256Type) ID() Type { return DECIMAL256 } +func (*Decimal256Type) Name() string { return "decimal256" } +func (*Decimal256Type) BitWidth() int { return 256 } +func (t *Decimal256Type) String() string { + return fmt.Sprintf("%s(%d, %d)", t.Name(), t.Precision, t.Scale) +} +func (t *Decimal256Type) Fingerprint() string { + return fmt.Sprintf("%s[%d,%d,%d]", typeFingerprint(t), t.BitWidth(), t.Precision, t.Scale) +} + +func (Decimal256Type) Layout() DataTypeLayout { + return DataTypeLayout{Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(Decimal128SizeBytes)}} Review Comment: Probably this needs to be fixed as follows? ```suggestion return DataTypeLayout{Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(Decimal256SizeBytes)}} ``` ########## go/arrow/scalar/scalar.go: ########## @@ -297,6 +298,8 @@ func (s *Decimal128) CastTo(to arrow.DataType) (Scalar, error) { switch to.ID() { case arrow.DECIMAL128: return NewDecimal128Scalar(s.Value, to), nil + case arrow.DECIMAL256: + return NewDecimal256Scalar(decimal256.FromDecimal128(s.Value), to), nil Review Comment: Same as below: should take into account differences in scale or precision. ########## go/arrow/decimal256/decimal256.go: ########## @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal256 + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/apache/arrow/go/v10/arrow/decimal128" + "github.com/apache/arrow/go/v10/arrow/internal/debug" +) + +const ( + MaxPrecision = 76 + MaxScale = 76 +) + +type Num struct { + // arr[0] is the lowest bits, arr[3] is the highest bits + arr [4]uint64 +} + +// New returns a new signed 256-bit integer value where x1 contains +// the highest bits with the rest of the values in order down to the +// lowest bits +// +// ie: New(1, 2, 3, 4) returns with the elements in little-endian order +// {4, 3, 2, 1} but each value is still represented as the native endianness +func New(x1, x2, x3, x4 uint64) Num { + return Num{[4]uint64{x4, x3, x2, x1}} +} + +func (n Num) Array() [4]uint64 { return n.arr } + +func (n Num) LowBits() uint64 { return n.arr[0] } + +func FromDecimal128(n decimal128.Num) Num { + var topBits uint64 + if n.Sign() < 0 { + topBits = math.MaxUint64 + } + return New(topBits, topBits, uint64(n.HighBits()), n.LowBits()) +} + +func FromU64(v uint64) Num { + return Num{[4]uint64{v, 0, 0, 0}} +} + +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, 0, 0, uint64(v)) + case v < 0: + return New(math.MaxUint64, math.MaxUint64, math.MaxUint64, uint64(v)) + default: + return Num{} + } +} + +func (n Num) Negate() Num { + var carry uint64 = 1 + for i := range n.arr { + n.arr[i] = ^n.arr[i] + carry + if n.arr[i] != 0 { + carry = 0 + } + } + return n +} + +func FromFloat32(v float32, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(float64(v), 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat32(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat32(v, prec, scale) +} + +func fromPositiveFloat32(v float32, prec, scale int32) (Num, error) { + var pscale float32 + if scale >= -76 && scale <= 76 { + pscale = float32PowersOfTen[scale+76] + } else { + pscale = float32(math.Pow10(int(scale))) + } + + v *= pscale + v = float32(math.RoundToEven(float64(v))) + maxabs := float32PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float32 + arr[3] = float32(math.Floor(math.Ldexp(float64(v), -192))) + v -= float32(math.Ldexp(float64(arr[3]), 192)) + arr[2] = float32(math.Floor(math.Ldexp(float64(v), -128))) + v -= float32(math.Ldexp(float64(arr[2]), 128)) + arr[1] = float32(math.Floor(math.Ldexp(float64(v), -64))) + v -= float32(math.Ldexp(float64(arr[1]), 64)) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func FromFloat64(v float64, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(v, 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat64(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat64(v, prec, scale) +} + +func fromPositiveFloat64(v float64, prec, scale int32) (Num, error) { + var pscale float64 + if scale >= -76 && scale <= 76 { + pscale = float64PowersOfTen[scale+76] + } else { + pscale = math.Pow10(int(scale)) + } + + v *= pscale + v = math.RoundToEven(v) + maxabs := float64PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float64 + arr[3] = math.Floor(math.Ldexp(v, -192)) + v -= math.Ldexp(arr[3], 192) + arr[2] = math.Floor(math.Ldexp(v, -128)) + v -= math.Ldexp(arr[2], 128) + arr[1] = math.Floor(math.Ldexp(v, -64)) + v -= math.Ldexp(arr[1], 64) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func (n Num) tofloat32Positive(scale int32) float32 { + const twoTo64 float32 = 1.8446744e+19 + if n.arr[3] != 0 || n.arr[2] != 0 { + return floatInf + } + x := float32(n.arr[1]) * twoTo64 + x += float32(n.arr[0]) + if scale >= -76 && scale <= 76 { + return x * float32PowersOfTen[-scale+76] + } + + return x * float32(math.Pow10(-int(scale))) +} + +func (n Num) tofloat64Positive(scale int32) float64 { + const ( + twoTo64 float64 = 1.8446744073709552e+19 + twoTo128 float64 = 3.402823669209385e+38 + twoTo192 float64 = 6.277101735386681e+57 + ) + + x := float64(n.arr[3]) * twoTo192 + x += float64(n.arr[2]) * twoTo128 + x += float64(n.arr[1]) * twoTo64 + x += float64(n.arr[0]) + + if scale >= -76 && scale <= 76 { + return x * float64PowersOfTen[-scale+76] + } + + return x * math.Pow10(-int(scale)) +} + +func (n Num) ToFloat32(scale int32) float32 { + if n.Sign() < 0 { + return -n.Negate().tofloat32Positive(scale) + } + return n.tofloat32Positive(scale) +} + +func (n Num) ToFloat64(scale int32) float64 { + if n.Sign() < 0 { + return -n.Negate().tofloat64Positive(scale) + } + return n.tofloat64Positive(scale) +} + +func (n Num) Sign() int { + if n == (Num{}) { + return 0 + } + return int(1 | (int64(n.arr[3]) >> 63)) +} + +func FromBigInt(v *big.Int) (n Num) { + bitlen := v.BitLen() + if bitlen > 256 { Review Comment: Shouldn't you account for the sign bit? ```suggestion if bitlen > 255 { ``` ########## go/arrow/decimal256/decimal256.go: ########## @@ -0,0 +1,569 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal256 + +import ( + "errors" + "fmt" + "math" + "math/big" + + "github.com/apache/arrow/go/v10/arrow/decimal128" + "github.com/apache/arrow/go/v10/arrow/internal/debug" +) + +const ( + MaxPrecision = 76 + MaxScale = 76 +) + +type Num struct { + // arr[0] is the lowest bits, arr[3] is the highest bits + arr [4]uint64 +} + +// New returns a new signed 256-bit integer value where x1 contains +// the highest bits with the rest of the values in order down to the +// lowest bits +// +// ie: New(1, 2, 3, 4) returns with the elements in little-endian order +// {4, 3, 2, 1} but each value is still represented as the native endianness +func New(x1, x2, x3, x4 uint64) Num { + return Num{[4]uint64{x4, x3, x2, x1}} +} + +func (n Num) Array() [4]uint64 { return n.arr } + +func (n Num) LowBits() uint64 { return n.arr[0] } + +func FromDecimal128(n decimal128.Num) Num { + var topBits uint64 + if n.Sign() < 0 { + topBits = math.MaxUint64 + } + return New(topBits, topBits, uint64(n.HighBits()), n.LowBits()) +} + +func FromU64(v uint64) Num { + return Num{[4]uint64{v, 0, 0, 0}} +} + +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, 0, 0, uint64(v)) + case v < 0: + return New(math.MaxUint64, math.MaxUint64, math.MaxUint64, uint64(v)) + default: + return Num{} + } +} + +func (n Num) Negate() Num { + var carry uint64 = 1 + for i := range n.arr { + n.arr[i] = ^n.arr[i] + carry + if n.arr[i] != 0 { + carry = 0 + } + } + return n +} + +func FromFloat32(v float32, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(float64(v), 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat32(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat32(v, prec, scale) +} + +func fromPositiveFloat32(v float32, prec, scale int32) (Num, error) { + var pscale float32 + if scale >= -76 && scale <= 76 { + pscale = float32PowersOfTen[scale+76] + } else { + pscale = float32(math.Pow10(int(scale))) + } + + v *= pscale + v = float32(math.RoundToEven(float64(v))) + maxabs := float32PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float32 + arr[3] = float32(math.Floor(math.Ldexp(float64(v), -192))) + v -= float32(math.Ldexp(float64(arr[3]), 192)) + arr[2] = float32(math.Floor(math.Ldexp(float64(v), -128))) + v -= float32(math.Ldexp(float64(arr[2]), 128)) + arr[1] = float32(math.Floor(math.Ldexp(float64(v), -64))) + v -= float32(math.Ldexp(float64(arr[1]), 64)) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744e+19, "bad conversion float32 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func FromFloat64(v float64, prec, scale int32) (Num, error) { + debug.Assert(prec > 0 && prec <= 76, "invalid precision for converting to decimal256") + + if math.IsInf(v, 0) { + return Num{}, fmt.Errorf("cannot convert %f to decimal256", v) + } + + if v < 0 { + dec, err := fromPositiveFloat64(-v, prec, scale) + if err != nil { + return dec, err + } + return dec.Negate(), nil + } + return fromPositiveFloat64(v, prec, scale) +} + +func fromPositiveFloat64(v float64, prec, scale int32) (Num, error) { + var pscale float64 + if scale >= -76 && scale <= 76 { + pscale = float64PowersOfTen[scale+76] + } else { + pscale = math.Pow10(int(scale)) + } + + v *= pscale + v = math.RoundToEven(v) + maxabs := float64PowersOfTen[prec+76] + if v <= -maxabs || v >= maxabs { + return Num{}, fmt.Errorf("cannot convert %f to decimal256(precision=%d, scale=%d): overflow", + v, prec, scale) + } + + var arr [4]float64 + arr[3] = math.Floor(math.Ldexp(v, -192)) + v -= math.Ldexp(arr[3], 192) + arr[2] = math.Floor(math.Ldexp(v, -128)) + v -= math.Ldexp(arr[2], 128) + arr[1] = math.Floor(math.Ldexp(v, -64)) + v -= math.Ldexp(arr[1], 64) + arr[0] = v + + debug.Assert(arr[3] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[3] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[2] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[2] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[1] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[1] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + debug.Assert(arr[0] >= 0, "bad conversion float64 to decimal256") + debug.Assert(arr[0] < 1.8446744073709552e+19, "bad conversion float64 to decimal256") // 2**64 + return Num{[4]uint64{uint64(arr[0]), uint64(arr[1]), uint64(arr[2]), uint64(arr[3])}}, nil +} + +func (n Num) tofloat32Positive(scale int32) float32 { + const twoTo64 float32 = 1.8446744e+19 + if n.arr[3] != 0 || n.arr[2] != 0 { + return floatInf + } + x := float32(n.arr[1]) * twoTo64 + x += float32(n.arr[0]) + if scale >= -76 && scale <= 76 { + return x * float32PowersOfTen[-scale+76] + } + + return x * float32(math.Pow10(-int(scale))) +} + +func (n Num) tofloat64Positive(scale int32) float64 { + const ( + twoTo64 float64 = 1.8446744073709552e+19 + twoTo128 float64 = 3.402823669209385e+38 + twoTo192 float64 = 6.277101735386681e+57 + ) + + x := float64(n.arr[3]) * twoTo192 + x += float64(n.arr[2]) * twoTo128 + x += float64(n.arr[1]) * twoTo64 + x += float64(n.arr[0]) + + if scale >= -76 && scale <= 76 { + return x * float64PowersOfTen[-scale+76] + } + + return x * math.Pow10(-int(scale)) +} + +func (n Num) ToFloat32(scale int32) float32 { + if n.Sign() < 0 { + return -n.Negate().tofloat32Positive(scale) + } + return n.tofloat32Positive(scale) +} + +func (n Num) ToFloat64(scale int32) float64 { + if n.Sign() < 0 { + return -n.Negate().tofloat64Positive(scale) + } + return n.tofloat64Positive(scale) +} + +func (n Num) Sign() int { + if n == (Num{}) { + return 0 + } + return int(1 | (int64(n.arr[3]) >> 63)) Review Comment: Is the purpose here to avoid branching? (otherwise why not a simple conditional?) ########## go/arrow/scalar/scalar.go: ########## @@ -311,6 +314,50 @@ func NewDecimal128Scalar(val decimal128.Num, typ arrow.DataType) *Decimal128 { return &Decimal128{scalar{typ, true}, val} } +type Decimal256 struct { + scalar + Value decimal256.Num +} + +func (s *Decimal256) value() interface{} { return s.Value } + +func (s *Decimal256) String() string { + if !s.Valid { + return "null" + } + val, err := s.CastTo(arrow.BinaryTypes.String) + if err != nil { + return "..." + } + return string(val.(*String).Value.Bytes()) +} + +func (s *Decimal256) equals(rhs Scalar) bool { + return s.Value == rhs.(*Decimal256).Value +} + +func (s *Decimal256) CastTo(to arrow.DataType) (Scalar, error) { + if !s.Valid { + return MakeNullScalar(to), nil + } + + switch to.ID() { + case arrow.DECIMAL256: + return NewDecimal256Scalar(s.Value, to), nil Review Comment: I think this is missing some casting of `s.Value` in case the scales don't match. And even if the scales match, should probably check that the value still fits in the new precision. ########## go/arrow/array/decimal256_test.go: ########## @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array_test + +import ( + "testing" + + "github.com/apache/arrow/go/v10/arrow" + "github.com/apache/arrow/go/v10/arrow/array" + "github.com/apache/arrow/go/v10/arrow/decimal256" + "github.com/apache/arrow/go/v10/arrow/memory" + "github.com/stretchr/testify/assert" +) + +func TestNewDecimal256Builder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + ab := array.NewDecimal256Builder(mem, &arrow.Decimal256Type{Precision: 10, Scale: 1}) + defer ab.Release() + + ab.Retain() + ab.Release() + + want := []decimal256.Num{ + decimal256.New(1, 1, 1, 1), + decimal256.New(2, 2, 2, 2), + decimal256.New(3, 3, 3, 3), + {}, + decimal256.FromI64(-5), + decimal256.FromI64(-6), + {}, + decimal256.FromI64(8), + decimal256.FromI64(9), + decimal256.FromI64(10), + } + valids := []bool{true, true, true, false, true, true, false, true, true, true} + + for i, valid := range valids { + switch { + case valid: + ab.Append(want[i]) + default: + ab.AppendNull() + } + } + + // check state of builder before NewDecimal256Array + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.NewArray().(*array.Decimal256) + a.Retain() + a.Release() + + // check state of builder after NewDecimal256Array + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), NewDecimal256Array did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), NewDecimal256Array did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), NewDecimal256Array did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + + assert.Equal(t, want, a.Values(), "unexpected Decimal256Values") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.Values(), 10, "unexpected length of Decimal256Values") + + a.Release() + ab.Append(decimal256.FromI64(7)) + ab.Append(decimal256.FromI64(8)) + + a = ab.NewDecimal256Array() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []decimal256.Num{decimal256.FromI64(7), decimal256.FromI64(8)}, a.Values()) + assert.Len(t, a.Values(), 2) Review Comment: Is it possible to also test the byte size of the buffers here? ########## go/arrow/datatype_fixedwidth.go: ########## @@ -529,6 +529,26 @@ func (Decimal128Type) Layout() DataTypeLayout { return DataTypeLayout{Buffers: []BufferSpec{SpecBitmap(), SpecFixedWidth(Decimal128SizeBytes)}} } +// Decimal128Type represents a fixed-size 128-bit decimal type. Review Comment: Obvious fix ```suggestion // Decimal256Type represents a fixed-size 256-bit decimal type. ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
