alvarowolfx commented on PR #421:
URL: https://github.com/apache/arrow-go/pull/421#issuecomment-3005699936
This is what I tried with `GetSchemaPayload` and `GetRecordBatchPayload`
(missing some err handling code). I get `Arrow Error: IOError: Invalid
flatbuffers message` from the BigQuery side.
```go
func writeToBQClient(schema *arrow.Schema, records []arrow.Record,
writeStreamName string, appendStream storagepb.BigQueryWrite_AppendRowsClient)
error {
// Serialize schema using IPC format (just the schema, no data)
var err error
var schemaBuf bytes.Buffer
schemaPayload := ipc.GetSchemaPayload(schema, memory.DefaultAllocator)
schemaMd := schemaPayload.Meta()
defer schemaMd.Release()
schemaBuf.Write(schemaMd.Bytes())
schemaPayload.SerializeBody(&schemaBuf)
schemaData := schemaBuf.Bytes()
// Serialize record batch using IPC format
var recordBuf bytes.Buffer
for _, record := range records {
p, err := ipc.GetRecordBatchPayload(record,
ipc.WithSchema(schema))
if err != nil {
slog.Error("failed to write Arrow record", "error", err)
return err
}
m := p.Meta()
defer m.Release()
recordBuf.Write(m.Bytes())
p.SerializeBody(&recordBuf)
}
recordData := recordBuf.Bytes()
slog.Info("Schema IPC data size", "bytes", len(schemaData))
slog.Info("Record IPC data size", "bytes", len(recordData))
request := &storagepb.AppendRowsRequest{
WriteStream: writeStreamName,
Rows: &storagepb.AppendRowsRequest_ArrowRows{
ArrowRows: &storagepb.AppendRowsRequest_ArrowData{
WriterSchema: &storagepb.ArrowSchema{
SerializedSchema: schemaData,
},
Rows: &storagepb.ArrowRecordBatch{
SerializedRecordBatch: recordData,
},
},
},
}
// Send the request
err = appendStream.Send(request)
if err != nil {
slog.Error("failed to send AppendRows request", "error", err)
return err
}
err = appendStream.CloseSend()
if err != nil {
slog.Error("failed to close AppendRows request: %v")
return err
}
return nil
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
