Hey,
Don't know if it's the official way but we have written our own proto to
BigQuery converter which works pretty well.
public static TableRow convertEventToTableRow(TableRow tableRow,
Message event) {
Map<Descriptors.FieldDescriptor, Object> fields = event.getAllFields();
for (Descriptors.FieldDescriptor field : fields.keySet()) {
tableRow = mapToBigQueryField(tableRow, field, fields.get(field));
}
return tableRow;
}
private static TableRow mapToBigQueryField(
TableRow tableRow, Descriptors.FieldDescriptor field, Object value) {
Descriptors.FieldDescriptor.JavaType fieldType = field.getJavaType();
switch (fieldType) {
case INT:
case LONG:
case FLOAT:
case DOUBLE:
case BOOLEAN:
return tableRow.set(field.getName(), value);
case BYTE_STRING:
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value,
x ->
Base64.getEncoder()
.encodeToString(
((ByteString)
x).toByteArray())));
} else {
return tableRow.set(
field.getName(),
Base64.getEncoder().encodeToString(((ByteString)
value).toByteArray()));
}
case ENUM:
if (field.isRepeated()) {
return tableRow.set(
field.getName(), processRepeatedField(value, x
-> x.toString()));
} else {
return tableRow.set(field.getName(), value.toString());
}
case STRING:
if (isUUIDField(field.getName())) {
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value, x ->
UUIDUtil.getBase64FromUUID((String) x)));
} else {
return tableRow.set(
field.getName(),
UUIDUtil.getBase64FromUUID((String) value));
}
} else {
return tableRow.set(field.getName(), value);
}
case MESSAGE:
switch (field.getMessageType().getFullName()) {
// Map well known message types that we have a
specific mapping for.
case "google.protobuf.Timestamp":
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value,
x ->
com.google.cloud.Timestamp.fromProto(
(Timestamp) x)
.toString()));
} else {
return tableRow.set(
field.getName(),
com.google.cloud.Timestamp.fromProto((Timestamp) value)
.toString());
}
case "xxx.xxx.ExactNumber":
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value, x ->
NumberUtils.toString((ExactNumber) x)));
} else {
return tableRow.set(
field.getName(),
NumberUtils.toString((ExactNumber) value));
}
case "google.protobuf.UInt64Value":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.UInt64Value.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((UInt64Value) value).getValue());
}
case "google.protobuf.Int32Value":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.Int32Value.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((Int32Value) value).getValue());
}
case "google.protobuf.DoubleValue":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.DoubleValue.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((DoubleValue) value).getValue());
}
case "google.protobuf.FloatValue":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.FloatValue.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((FloatValue) value).getValue());
}
case "google.protobuf.Int64Value":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.Int64Value.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((Int64Value) value).getValue());
}
case "google.protobuf.UInt32Value":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.UInt32Value.getDefaultInstance())) {
value = 0;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((UInt32Value) value).getValue());
}
case "google.protobuf.BoolValue":
if (field.hasDefaultValue()) break;
else if
(value.equals(com.google.protobuf.BoolValue.getDefaultInstance())) {
value = false;
return tableRow.set(field.getName(), value);
} else {
return tableRow.set(field.getName(),
((BoolValue) value).getValue());
}
case "google.protobuf.StringValue":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.StringValue.getDefaultInstance())) {
value = "";
return tableRow.set(field.getName(), value);
} else if (isUUIDField(field.getName())) {
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value,
x ->
UUIDUtil.getBase64FromUUID(
((StringValue) x).getValue())));
} else {
return tableRow.set(
field.getName(),
UUIDUtil.getBase64FromUUID(
((StringValue) value).getValue()));
}
} else if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(value, x ->
((StringValue) x).getValue()));
} else {
return tableRow.set(field.getName(),
((StringValue) value).getValue());
}
case "google.protobuf.BytesValue":
if (field.hasDefaultValue()) {
break;
} else if (value.equals(
com.google.protobuf.BytesValue.getDefaultInstance())) {
value = ByteString.EMPTY;
return tableRow.set(field.getName(), value);
} else if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value,
x ->
Base64.getEncoder()
.encodeToString(
((BytesValue) x)
.getValue()
.toByteArray())));
} else {
return tableRow.set(
field.getName(),
Base64.getEncoder()
.encodeToString(
((BytesValue)
value).getValue().toByteArray()));
}
default:
if (field.isRepeated()) {
return tableRow.set(
field.getName(),
processRepeatedField(
value,
x ->
convertEventToTableRow(
new
TableRow(), (Message) x)));
} else {
return tableRow.set(
field.getName(),
convertEventToTableRow(new TableRow(),
(Message) value));
}
}
default:
throw new IllegalArgumentException(
field.getFullName() + " has an unsupported type "
+ field.getType());
}
}
On Wed, Jul 8, 2020 at 4:40 PM Kaymak, Tobias <[email protected]>
wrote:
> As a workaround I am currently using the following code to generate a
> TableRow object from a Java Protobuf class - as I am facing a problem with
> Beam schemas (
> https://www.mail-archive.com/[email protected]/msg05799.html).
>
> It relies on the TableRowJsonCoder:
>
> String json = JsonFormat.printer().omittingInsignificantWhitespace()
> .preservingProtoFieldNames().print(article.toBuilder());
> InputStream inputStream = new
> ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
>
> TableRow tableRow = TableRowJsonCoder.of().decode(inputStream,
> Coder.Context.OUTER);
>
> However, the usage of Coder.Context is deprecated - I've tried to simply
> use the decode(), but that defaults to Context.NESTED.
>
> What is the correct way of doing this?
>
> Best,
> Tobi
>
