This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new d08c7b29 fix(arrow/csv): preserve decimal precision (#1022)
d08c7b29 is described below
commit d08c7b299a2e9d6156e6065de1d084cc0c84ff2e
Author: Minh Vu <[email protected]>
AuthorDate: Mon Jul 27 19:07:59 2026 +0200
fix(arrow/csv): preserve decimal precision (#1022)
### Rationale for this change
The CSV writer converts decimal values through math.Pow10 and big.Float.
That changes valid Decimal128 and Decimal256 values for many scales and
drops significant trailing zeros from fixed-scale output.
### What changes are included in this PR?
* Format Decimal128 and Decimal256 values directly with their exact
ToString implementations.
* Preserve the declared scale, including trailing zeros and negative
scales.
* Update existing zero-value expectations to retain their two decimal
places.
### Are these changes tested?
Yes. The regression test covers a high-precision Decimal128 with a
positive scale and a Decimal256 with a negative scale. All CSV tests and
the assertion build pass with the required Parquet test data.
---
arrow/csv/transformer.go | 12 ++----------
arrow/csv/writer_test.go | 30 ++++++++++++++++++++++++++++--
2 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/arrow/csv/transformer.go b/arrow/csv/transformer.go
index 0c663c7f..807050c8 100644
--- a/arrow/csv/transformer.go
+++ b/arrow/csv/transformer.go
@@ -21,8 +21,6 @@ import (
"encoding/base64"
"encoding/csv"
"fmt"
- "math"
- "math/big"
"strconv"
"github.com/apache/arrow-go/v18/arrow"
@@ -197,13 +195,10 @@ func (w *Writer) transformColToStringArr(typ
arrow.DataType, col arrow.Array, st
case *arrow.Decimal128Type:
fieldType := typ.(*arrow.Decimal128Type)
scale := fieldType.Scale
- precision := fieldType.Precision
arr := col.(*array.Decimal128)
for i := 0; i < arr.Len(); i++ {
if arr.IsValid(i) {
- f :=
(&big.Float{}).SetInt(arr.Value(i).BigInt())
- f.Quo(f, big.NewFloat(math.Pow10(int(scale))))
- res[i] = f.Text('g', int(precision))
+ res[i] = arr.Value(i).ToString(scale)
} else {
res[i] = w.nullValue
}
@@ -211,13 +206,10 @@ func (w *Writer) transformColToStringArr(typ
arrow.DataType, col arrow.Array, st
case *arrow.Decimal256Type:
fieldType := typ.(*arrow.Decimal256Type)
scale := fieldType.Scale
- precision := fieldType.Precision
arr := col.(*array.Decimal256)
for i := 0; i < arr.Len(); i++ {
if arr.IsValid(i) {
- f :=
(&big.Float{}).SetInt(arr.Value(i).BigInt())
- f.Quo(f, big.NewFloat(math.Pow10(int(scale))))
- res[i] = f.Text('g', int(precision))
+ res[i] = arr.Value(i).ToString(scale)
} else {
res[i] = w.nullValue
}
diff --git a/arrow/csv/writer_test.go b/arrow/csv/writer_test.go
index e227b3b7..1701f999 100644
--- a/arrow/csv/writer_test.go
+++ b/arrow/csv/writer_test.go
@@ -145,14 +145,14 @@ var (
fullData = [][]string{
{"bool", "i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64",
"f16", "f32", "f64", "str", "large_str", "ts_s", "d32", "d64", "dec128",
"dec256", "list(i64)", "large_list(i64)", "fixed_size_list(i64)", "binary",
"large_binary", "fixed_size_binary", "uuid", "null"},
{"true", "-1", "-1", "-1", "-1", "0", "0", "0", "0", "0", "0",
"0", "str-0", "str-0", "2014-07-28 15:04:05", "2017-05-18", "2028-04-26",
"-123.45", "-123.45", "{1,2,3}", "{1,2,3}", "{1,2,3}", "AAEC", "AAEC", "AAEC",
"00000000-0000-0000-0000-000000000001", nullVal},
- {"false", "0", "0", "0", "0", "1", "1", "1", "1",
"0.099975586", "0.1", "0.1", "str-1", "str-1", "2016-09-08 15:04:05",
"2022-11-08", "2031-06-28", "0", "0", "{4,5,6}", "{4,5,6}", "{4,5,6}", "AwQF",
"AwQF", "AwQF", "00000000-0000-0000-0000-000000000002", nullVal},
+ {"false", "0", "0", "0", "0", "1", "1", "1", "1",
"0.099975586", "0.1", "0.1", "str-1", "str-1", "2016-09-08 15:04:05",
"2022-11-08", "2031-06-28", "0.00", "0.00", "{4,5,6}", "{4,5,6}", "{4,5,6}",
"AwQF", "AwQF", "AwQF", "00000000-0000-0000-0000-000000000002", nullVal},
{"true", "1", "1", "1", "1", "2", "2", "2", "2", "0.19995117",
"0.2", "0.2", "str-2", "str-2", "2021-09-18 15:04:05", "2025-08-04",
"2034-08-28", "123.45", "123.45", "{7,8,9}", "{7,8,9}", "{7,8,9}", "", "",
"AAAA", "00000000-0000-0000-0000-000000000003", nullVal},
{nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal},
}
bananaData = [][]string{
{"bool", "i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64",
"f16", "f32", "f64", "str", "large_str", "ts_s", "d32", "d64", "dec128",
"dec256", "list(i64)", "large_list(i64)", "fixed_size_list(i64)", "binary",
"large_binary", "fixed_size_binary", "uuid", "null"},
{"BANANA", "-1", "-1", "-1", "-1", "0", "0", "0", "0", "0",
"0", "0", "str-0", "str-0", "2014-07-28 15:04:05", "2017-05-18", "2028-04-26",
"-123.45", "-123.45", "{1,2,3}", "{1,2,3}", "{1,2,3}", "AAEC", "AAEC", "AAEC",
"00000000-0000-0000-0000-000000000001", nullVal},
- {"MANGO", "0", "0", "0", "0", "1", "1", "1", "1",
"0.099975586", "0.1", "0.1", "str-1", "str-1", "2016-09-08 15:04:05",
"2022-11-08", "2031-06-28", "0", "0", "{4,5,6}", "{4,5,6}", "{4,5,6}", "AwQF",
"AwQF", "AwQF", "00000000-0000-0000-0000-000000000002", nullVal},
+ {"MANGO", "0", "0", "0", "0", "1", "1", "1", "1",
"0.099975586", "0.1", "0.1", "str-1", "str-1", "2016-09-08 15:04:05",
"2022-11-08", "2031-06-28", "0.00", "0.00", "{4,5,6}", "{4,5,6}", "{4,5,6}",
"AwQF", "AwQF", "AwQF", "00000000-0000-0000-0000-000000000002", nullVal},
{"BANANA", "1", "1", "1", "1", "2", "2", "2", "2",
"0.19995117", "0.2", "0.2", "str-2", "str-2", "2021-09-18 15:04:05",
"2025-08-04", "2034-08-28", "123.45", "123.45", "{7,8,9}", "{7,8,9}",
"{7,8,9}", "", "", "AAAA", "00000000-0000-0000-0000-000000000003", nullVal},
{nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal, nullVal,
nullVal, nullVal, nullVal, nullVal},
}
@@ -437,6 +437,32 @@ func BenchmarkWrite(b *testing.B) {
}
}
+func TestCSVWriterPreservesDecimalScaleAndPrecision(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ type128 := &arrow.Decimal128Type{Precision: 38, Scale: 10}
+ type256 := &arrow.Decimal256Type{Precision: 76, Scale: -10}
+ builder := array.NewRecordBuilder(mem, arrow.NewSchema([]arrow.Field{
+ {Name: "decimal128", Type: type128},
+ {Name: "decimal256", Type: type256},
+ }, nil))
+ defer builder.Release()
+
+ value128 := decimal128.FromU64(12345678901234567890)
+ value256 := decimal256.FromU64(12345678901234567890)
+ builder.Field(0).(*array.Decimal128Builder).Append(value128)
+ builder.Field(1).(*array.Decimal256Builder).Append(value256)
+ record := builder.NewRecordBatch()
+ defer record.Release()
+
+ var output bytes.Buffer
+ writer := csv.NewWriter(&output, record.Schema())
+ require.NoError(t, writer.Write(record))
+ require.NoError(t, writer.Flush())
+ assert.Equal(t,
value128.ToString(type128.Scale)+","+value256.ToString(type256.Scale)+"\n",
output.String())
+}
+
// TestParquetTestingCSVWriter tests that the CSV writer successfully convert
arrow/parquet-testing files to CSV
func TestParquetTestingCSVWriter(t *testing.T) {
dir := os.Getenv("PARQUET_TEST_DATA")
