This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 0b0828bff perf(rest): parse numeric scan partition values directly
(#2115)
0b0828bff is described below
commit 0b0828bffd8ba7f50e18cbafa59d6a4b8e5df2f6
Author: Minh Vu <[email protected]>
AuthorDate: Tue Oct 6 10:28:44 2026 +0200
perf(rest): parse numeric scan partition values directly (#2115)
* perf(rest): parse numeric scan partition values directly
* fix(rest): preserve quoted numeric partition values
---
catalog/rest/scan_task_decoder.go | 35 +++++++----
catalog/rest/scan_task_decoder_bench_test.go | 71 ++++++++++++++++++++++-
catalog/rest/scan_task_decoder_test.go | 87 ++++++++++++++++++++++++++++
3 files changed, 180 insertions(+), 13 deletions(-)
diff --git a/catalog/rest/scan_task_decoder.go
b/catalog/rest/scan_task_decoder.go
index ba7f9aa7e..82e833647 100644
--- a/catalog/rest/scan_task_decoder.go
+++ b/catalog/rest/scan_task_decoder.go
@@ -26,6 +26,7 @@ import (
"math"
"reflect"
"slices"
+ "strconv"
"github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/table"
@@ -763,13 +764,18 @@ func decodePartitionLiteral(raw json.RawMessage, typ
iceberg.Type) (iceberg.Lite
}
func decodeJSONInteger(raw json.RawMessage, bitSize int) (int64, error) {
- dec := json.NewDecoder(bytes.NewReader(raw))
- dec.UseNumber()
- var value json.Number
- if err := dec.Decode(&value); err != nil {
- return 0, fmt.Errorf("invalid integer value: %w", err)
+ value := bytes.Trim(raw, " \t\r\n")
+ if !json.Valid(value) {
+ return 0, fmt.Errorf("invalid integer value: invalid JSON
number %q", raw)
}
- parsed, err := value.Int64()
+ if len(value) > 0 && value[0] == '"' {
+ var number json.Number
+ if err := json.Unmarshal(value, &number); err != nil {
+ return 0, fmt.Errorf("invalid integer value: %w", err)
+ }
+ value = []byte(number)
+ }
+ parsed, err := strconv.ParseInt(string(value), 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid integer value %q: %w", value, err)
}
@@ -781,13 +787,18 @@ func decodeJSONInteger(raw json.RawMessage, bitSize int)
(int64, error) {
}
func decodeJSONFloat(raw json.RawMessage, bitSize int) (float64, error) {
- dec := json.NewDecoder(bytes.NewReader(raw))
- dec.UseNumber()
- var value json.Number
- if err := dec.Decode(&value); err != nil {
- return 0, fmt.Errorf("invalid floating-point value: %w", err)
+ value := bytes.Trim(raw, " \t\r\n")
+ if !json.Valid(value) {
+ return 0, fmt.Errorf("invalid floating-point value: invalid
JSON number %q", raw)
+ }
+ if len(value) > 0 && value[0] == '"' {
+ var number json.Number
+ if err := json.Unmarshal(value, &number); err != nil {
+ return 0, fmt.Errorf("invalid floating-point value:
%w", err)
+ }
+ value = []byte(number)
}
- parsed, err := value.Float64()
+ parsed, err := strconv.ParseFloat(string(value), 64)
if err != nil {
return 0, fmt.Errorf("invalid floating-point value %q: %w",
value, err)
}
diff --git a/catalog/rest/scan_task_decoder_bench_test.go
b/catalog/rest/scan_task_decoder_bench_test.go
index c7130484d..6ac8466db 100644
--- a/catalog/rest/scan_task_decoder_bench_test.go
+++ b/catalog/rest/scan_task_decoder_bench_test.go
@@ -18,11 +18,16 @@
package rest
import (
+ "encoding/json"
"fmt"
"testing"
)
-var decodeScanTasksBenchmarkSink int
+var (
+ decodeScanTasksBenchmarkSink int
+ decodeJSONIntegerBenchmarkSink int64
+ decodeJSONFloatBenchmarkSink float64
+)
func BenchmarkDecodeScanTasksDeletionVectors(b *testing.B) {
metadata := newScanTaskDecoderMetadata()
@@ -109,3 +114,67 @@ func BenchmarkDecodeScanTasks(b *testing.B) {
})
}
}
+
+func BenchmarkDecodeJSONInteger(b *testing.B) {
+ raw := json.RawMessage("12345")
+ for _, bitSize := range []int{32, 64} {
+ b.Run(fmt.Sprintf("int%d", bitSize), func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ value, err := decodeJSONInteger(raw, bitSize)
+ if err != nil {
+ b.Fatal(err)
+ }
+ decodeJSONIntegerBenchmarkSink = value
+ }
+ })
+ }
+}
+
+func BenchmarkDecodeJSONFloat(b *testing.B) {
+ raw := json.RawMessage("12345.25")
+ for _, bitSize := range []int{32, 64} {
+ b.Run(fmt.Sprintf("float%d", bitSize), func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ value, err := decodeJSONFloat(raw, bitSize)
+ if err != nil {
+ b.Fatal(err)
+ }
+ decodeJSONFloatBenchmarkSink = value
+ }
+ })
+ }
+}
+
+func BenchmarkDecodeJSONNumberForms(b *testing.B) {
+ for _, tc := range []struct {
+ name, raw string
+ floating bool
+ }{
+ {name: "int64_exact", raw: "9007199254740993"},
+ {name: "quoted_int64", raw: `"9007199254740993"`},
+ {name: "exponent", raw: "-1.25e2", floating: true},
+ {name: "quoted_exponent", raw: `"-1.25e2"`, floating: true},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ raw := json.RawMessage(tc.raw)
+ b.ReportAllocs()
+ for b.Loop() {
+ if tc.floating {
+ value, err := decodeJSONFloat(raw, 64)
+ if err != nil {
+ b.Fatal(err)
+ }
+ decodeJSONFloatBenchmarkSink = value
+ } else {
+ value, err := decodeJSONInteger(raw, 64)
+ if err != nil {
+ b.Fatal(err)
+ }
+ decodeJSONIntegerBenchmarkSink = value
+ }
+ }
+ })
+ }
+}
diff --git a/catalog/rest/scan_task_decoder_test.go
b/catalog/rest/scan_task_decoder_test.go
index b796c4a17..e4d624d7a 100644
--- a/catalog/rest/scan_task_decoder_test.go
+++ b/catalog/rest/scan_task_decoder_test.go
@@ -630,6 +630,93 @@ func TestDecodePartitionLiteralRejectsInvalidValues(t
*testing.T) {
}
}
+func TestDecodeJSONInteger(t *testing.T) {
+ t.Parallel()
+
+ for _, tt := range []struct {
+ name string
+ raw string
+ bitSize int
+ want int64
+ wantErr string
+ }{
+ {name: "int64 max", raw: `9223372036854775807`, bitSize: 64,
want: math.MaxInt64},
+ {name: "int64 min with whitespace", raw: ` -9223372036854775808
`, bitSize: 64, want: math.MinInt64},
+ {name: "int32 max", raw: `2147483647`, bitSize: 32, want:
math.MaxInt32},
+ {name: "int32 min", raw: `-2147483648`, bitSize: 32, want:
math.MinInt32},
+ {name: "above float precision", raw: `9007199254740993`,
bitSize: 64, want: 9007199254740993},
+ {name: "quoted integer", raw: `"34"`, bitSize: 32, want: 34},
+ {name: "escaped integer", raw: `"\u0033\u0034"`, bitSize: 64,
want: 34},
+ {name: "quoted overflow", raw: `"9223372036854775808"`,
bitSize: 64, wantErr: "invalid integer value"},
+ {name: "invalid quoted number", raw: `"not-a-number"`, bitSize:
64, wantErr: "invalid integer value"},
+ {name: "non JSON whitespace", raw: "\u00a01\u00a0", bitSize:
64, wantErr: "invalid JSON number"},
+ {name: "fraction", raw: `1.5`, bitSize: 64, wantErr: "invalid
integer value"},
+ {name: "exponent", raw: `1e2`, bitSize: 64, wantErr: "invalid
integer value"},
+ {name: "int64 overflow", raw: `9223372036854775808`, bitSize:
64, wantErr: "invalid integer value"},
+ {name: "int32 overflow", raw: `2147483648`, bitSize: 32,
wantErr: "outside int32 range"},
+ {name: "leading plus", raw: `+1`, bitSize: 64, wantErr:
"invalid JSON number"},
+ {name: "leading zero", raw: `01`, bitSize: 64, wantErr:
"invalid JSON number"},
+ {name: "trailing value", raw: `1 2`, bitSize: 64, wantErr:
"invalid JSON number"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ got, err := decodeJSONInteger(json.RawMessage(tt.raw),
tt.bitSize)
+ if tt.wantErr != "" {
+ require.ErrorContains(t, err, tt.wantErr)
+
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestDecodeJSONFloat(t *testing.T) {
+ t.Parallel()
+
+ for _, tt := range []struct {
+ name string
+ raw string
+ bitSize int
+ want float64
+ wantSignbit bool
+ wantErr string
+ }{
+ {name: "exponent with whitespace", raw: ` -1.25e2 `, bitSize:
64, want: -125},
+ {name: "negative zero", raw: `-0`, bitSize: 64, wantSignbit:
true},
+ {name: "float32 range", raw: `3.4028234e38`, bitSize: 32, want:
3.4028234e38},
+ {name: "float32 overflow", raw: `3.5e38`, bitSize: 32, wantErr:
"outside float32 range"},
+ {name: "quoted float", raw: `"1.25"`, bitSize: 64, want: 1.25},
+ {name: "escaped float", raw: `"\u0031.25"`, bitSize: 32, want:
1.25},
+ {name: "quoted exponent", raw: `"-1.25e2"`, bitSize: 64, want:
-125},
+ {name: "quoted negative zero", raw: `"-0"`, bitSize: 64,
wantSignbit: true},
+ {name: "quoted overflow", raw: `"1e999"`, bitSize: 64, wantErr:
"invalid floating-point value"},
+ {name: "invalid quoted number", raw: `"NaN"`, bitSize: 64,
wantErr: "invalid floating-point value"},
+ {name: "non JSON whitespace", raw: "\u00a01.25\u00a0", bitSize:
64, wantErr: "invalid JSON number"},
+ {name: "overflow", raw: `1e999`, bitSize: 64, wantErr: "invalid
floating-point value"},
+ {name: "leading plus", raw: `+1.5`, bitSize: 64, wantErr:
"invalid JSON number"},
+ {name: "trailing value", raw: `1.5 2`, bitSize: 64, wantErr:
"invalid JSON number"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ got, err := decodeJSONFloat(json.RawMessage(tt.raw),
tt.bitSize)
+ if tt.wantErr != "" {
+ require.ErrorContains(t, err, tt.wantErr)
+
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tt.want, got)
+ if tt.wantSignbit {
+ assert.True(t, math.Signbit(got))
+ }
+ })
+ }
+}
+
func validScanTasksWire() ScanTasks {
data := RESTContentFile{
SpecID: 7,
