zeroshade commented on code in PR #35769:
URL: https://github.com/apache/arrow/pull/35769#discussion_r1238716504
##########
go/arrow/internal/arrjson/arrjson.go:
##########
@@ -2179,3 +2230,113 @@ func durationToJSON(arr *array.Duration) []interface{} {
}
return o
}
+
+func variadicBuffersFromJSON(bufs []string) []*memory.Buffer {
+ out := make([]*memory.Buffer, len(bufs))
+ for i, data := range bufs {
+ rawData, err := hex.DecodeString(data)
+ if err != nil {
+ panic(err)
+ }
+
+ out[i] = memory.NewBufferBytes(rawData)
+ }
+ return out
+}
+
+func variadicBuffersToJSON(bufs []*memory.Buffer) []string {
+ out := make([]string, len(bufs))
+ for i, data := range bufs {
+ out[i] = strings.ToUpper(hex.EncodeToString(data.Bytes()))
+ }
+ return out
+}
+
+func stringHeadersFromJSON(mem memory.Allocator, isBinary bool, data
[]interface{}) *memory.Buffer {
+ buf := memory.NewResizableBuffer(mem)
+ buf.Resize(arrow.StringHeaderTraits.BytesRequired(len(data)))
+
+ values := arrow.StringHeaderTraits.CastFromBytes(buf.Bytes())
+
+ for i, d := range data {
+ switch v := d.(type) {
+ case nil:
+ continue
+ case map[string]interface{}:
+ if inlined, ok := v["INLINED"]; ok {
+ if isBinary {
+ val, err :=
hex.DecodeString(inlined.(string))
+ if err != nil {
+ panic(fmt.Errorf("could not
decode %v: %v", inlined, err))
+ }
+ values[i].SetBytes(val)
+ } else {
+ values[i].SetString(inlined.(string))
+ }
+ continue
+ }
+
+ idx, offset := v["BUFFER_INDEX"].(json.Number),
v["OFFSET"].(json.Number)
+ bufIdx, err := idx.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ bufOffset, err := offset.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ values[i].SetIndexOffset(uint32(bufIdx),
uint32(bufOffset))
+ prefix, err := hex.DecodeString(v["PREFIX"].(string))
+ if err != nil {
+ panic(err)
+ }
+ sz, err := v["SIZE"].(json.Number).Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ rawData := make([]byte, sz)
+ copy(rawData, prefix)
+ values[i].SetBytes(rawData)
+ }
+ }
+ return buf
+}
+
+func stringHeadersToJSON(arr array.ViewLike, isBinary bool) []interface{} {
+ type StringHeader struct {
+ Size int `json:"SIZE"`
+ Prefix *string `json:"PREFIX,omitempty"`
+ BufferIdx *int `json:"BUFFER_INDEX,omitempty"`
+ BufferOff *int `json:"OFFSET,omitempty"`
+ Inlined *string `json:"INLINED,omitempty"`
+ }
+
+ o := make([]interface{}, arr.Len())
+ for i := range o {
+ hdr := arr.ValueHeader(i)
+ if hdr.IsInline() {
+ data := hdr.InlineData()
+ if isBinary {
+ data =
strings.ToUpper(hex.EncodeToString(hdr.InlineBytes()))
Review Comment:
The spec for the integration test is to specify that the binary data should
be upper-cased hex characters. @bkietz can comment on whether or not the
ToUpper is necessary and if we want to relax the spec.
##########
go/arrow/internal/arrjson/arrjson.go:
##########
@@ -2179,3 +2230,113 @@ func durationToJSON(arr *array.Duration) []interface{} {
}
return o
}
+
+func variadicBuffersFromJSON(bufs []string) []*memory.Buffer {
+ out := make([]*memory.Buffer, len(bufs))
+ for i, data := range bufs {
+ rawData, err := hex.DecodeString(data)
+ if err != nil {
+ panic(err)
+ }
+
+ out[i] = memory.NewBufferBytes(rawData)
+ }
+ return out
+}
+
+func variadicBuffersToJSON(bufs []*memory.Buffer) []string {
+ out := make([]string, len(bufs))
+ for i, data := range bufs {
+ out[i] = strings.ToUpper(hex.EncodeToString(data.Bytes()))
+ }
+ return out
+}
+
+func stringHeadersFromJSON(mem memory.Allocator, isBinary bool, data
[]interface{}) *memory.Buffer {
+ buf := memory.NewResizableBuffer(mem)
+ buf.Resize(arrow.StringHeaderTraits.BytesRequired(len(data)))
+
+ values := arrow.StringHeaderTraits.CastFromBytes(buf.Bytes())
+
+ for i, d := range data {
+ switch v := d.(type) {
+ case nil:
+ continue
+ case map[string]interface{}:
+ if inlined, ok := v["INLINED"]; ok {
+ if isBinary {
+ val, err :=
hex.DecodeString(inlined.(string))
+ if err != nil {
+ panic(fmt.Errorf("could not
decode %v: %v", inlined, err))
+ }
+ values[i].SetBytes(val)
+ } else {
+ values[i].SetString(inlined.(string))
+ }
+ continue
+ }
+
+ idx, offset := v["BUFFER_INDEX"].(json.Number),
v["OFFSET"].(json.Number)
+ bufIdx, err := idx.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ bufOffset, err := offset.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ values[i].SetIndexOffset(uint32(bufIdx),
uint32(bufOffset))
+ prefix, err := hex.DecodeString(v["PREFIX"].(string))
+ if err != nil {
+ panic(err)
+ }
+ sz, err := v["SIZE"].(json.Number).Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ rawData := make([]byte, sz)
+ copy(rawData, prefix)
+ values[i].SetBytes(rawData)
+ }
+ }
+ return buf
+}
+
+func stringHeadersToJSON(arr array.ViewLike, isBinary bool) []interface{} {
+ type StringHeader struct {
+ Size int `json:"SIZE"`
+ Prefix *string `json:"PREFIX,omitempty"`
+ BufferIdx *int `json:"BUFFER_INDEX,omitempty"`
+ BufferOff *int `json:"OFFSET,omitempty"`
+ Inlined *string `json:"INLINED,omitempty"`
+ }
+
+ o := make([]interface{}, arr.Len())
+ for i := range o {
+ hdr := arr.ValueHeader(i)
+ if hdr.IsInline() {
+ data := hdr.InlineData()
+ if isBinary {
+ data =
strings.ToUpper(hex.EncodeToString(hdr.InlineBytes()))
+ }
+ o[i] = StringHeader{
+ Size: hdr.Len(),
+ Inlined: &data,
+ }
+ } else {
+ idx, off := int(hdr.BufferIndex()),
int(hdr.BufferOffset())
+ prefix := hdr.Prefix()
+ encodedPrefix :=
strings.ToUpper(hex.EncodeToString(prefix[:]))
Review Comment:
Let's keep this discussion to
https://github.com/apache/arrow/pull/35769#discussion_r1238716504
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
