bkietz commented on code in PR #35769:
URL: https://github.com/apache/arrow/pull/35769#discussion_r1283613027
##########
go/arrow/internal/arrjson/arrjson.go:
##########
@@ -2179,3 +2230,113 @@ func durationToJSON(arr *array.Duration) []interface{} {
}
return o
}
+
+func variadicBuffersFromJSON(bufs []string) []*memory.Buffer {
+ out := make([]*memory.Buffer, len(bufs))
+ for i, data := range bufs {
+ rawData, err := hex.DecodeString(data)
+ if err != nil {
+ panic(err)
+ }
+
+ out[i] = memory.NewBufferBytes(rawData)
+ }
+ return out
+}
+
+func variadicBuffersToJSON(bufs []*memory.Buffer) []string {
+ out := make([]string, len(bufs))
+ for i, data := range bufs {
+ out[i] = strings.ToUpper(hex.EncodeToString(data.Bytes()))
+ }
+ return out
+}
+
+func stringHeadersFromJSON(mem memory.Allocator, isBinary bool, data
[]interface{}) *memory.Buffer {
+ buf := memory.NewResizableBuffer(mem)
+ buf.Resize(arrow.StringHeaderTraits.BytesRequired(len(data)))
+
+ values := arrow.StringHeaderTraits.CastFromBytes(buf.Bytes())
+
+ for i, d := range data {
+ switch v := d.(type) {
+ case nil:
+ continue
+ case map[string]interface{}:
+ if inlined, ok := v["INLINED"]; ok {
+ if isBinary {
+ val, err :=
hex.DecodeString(inlined.(string))
+ if err != nil {
+ panic(fmt.Errorf("could not
decode %v: %v", inlined, err))
+ }
+ values[i].SetBytes(val)
+ } else {
+ values[i].SetString(inlined.(string))
+ }
+ continue
+ }
+
+ idx, offset := v["BUFFER_INDEX"].(json.Number),
v["OFFSET"].(json.Number)
+ bufIdx, err := idx.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ bufOffset, err := offset.Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ values[i].SetIndexOffset(uint32(bufIdx),
uint32(bufOffset))
+ prefix, err := hex.DecodeString(v["PREFIX"].(string))
+ if err != nil {
+ panic(err)
+ }
+ sz, err := v["SIZE"].(json.Number).Int64()
+ if err != nil {
+ panic(err)
+ }
+
+ rawData := make([]byte, sz)
+ copy(rawData, prefix)
+ values[i].SetBytes(rawData)
+ }
+ }
+ return buf
+}
+
+func stringHeadersToJSON(arr array.ViewLike, isBinary bool) []interface{} {
+ type StringHeader struct {
+ Size int `json:"SIZE"`
+ Prefix *string `json:"PREFIX,omitempty"`
+ BufferIdx *int `json:"BUFFER_INDEX,omitempty"`
+ BufferOff *int `json:"OFFSET,omitempty"`
+ Inlined *string `json:"INLINED,omitempty"`
+ }
+
+ o := make([]interface{}, arr.Len())
+ for i := range o {
+ hdr := arr.ValueHeader(i)
+ if hdr.IsInline() {
+ data := hdr.InlineData()
+ if isBinary {
+ data =
strings.ToUpper(hex.EncodeToString(hdr.InlineBytes()))
Review Comment:
I think relaxing the spec is not a trivial change; it'd require all
implementers to update how they consume binary data. That said, although it'd
be a lot of work updating the integration JSON (perhaps even including a real
JSON schema :scream:) is definitely worth a follow up.
See also:
- https://github.com/apache/arrow/pull/7110
- https://github.com/apache/arrow/issues/34898
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
