ahmedabu98 commented on code in PR #40998:
URL: https://github.com/apache/beam/pull/40998#discussion_r4590526193
##########
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java:
##########
@@ -684,24 +692,62 @@ private static void addIcebergValue(
field.getType().getRowSchema(),
"Corrupted schema: Row type did not have associated nested
schema.");
if (icebergValue instanceof Record) {
- rowBuilder.addValue(icebergRecordToBeamRow(nestedSchema, (Record)
icebergValue));
+ values.add(icebergRecordToBeamRow(nestedSchema, (Record)
icebergValue));
} else if (icebergValue instanceof StructLike) {
- rowBuilder.addValue(structToRow(nestedSchema, (StructLike)
icebergValue));
+ values.add(structToRow(nestedSchema, (StructLike) icebergValue));
} else {
throw new UnsupportedOperationException(
"Unsupported row type: " + icebergValue.getClass());
}
break;
case LOGICAL_TYPE:
- rowBuilder.addValue(getLogicalTypeValue(icebergValue,
field.getType()));
+ values.add(getLogicalTypeValue(icebergValue, field.getType()));
break;
default:
throw new UnsupportedOperationException(
"Unsupported Beam type: " + field.getType().getTypeName());
}
}
- private static DateTime getBeamDateTimeValue(Object icebergValue) {
+ /**
+ * Copies a list or map, or one of its elements, the way {@link
Row.Builder#build()} would. A null
+ * fails where the Beam type is not nullable, and {@code binary}, which
Iceberg reads as a {@link
+ * ByteBuffer}, becomes byte[].
+ */
+ private static @Nullable Object toBeamNestedValue(
+ Schema.FieldType type, @Nullable Object value, String fieldName) {
+ if (value == null) {
+ checkArgument(
+ type.getNullable(), "Received null value for %s in field '%s'.",
type, fieldName);
+ return null;
+ }
+ switch (type.getTypeName()) {
+ case BYTES:
+ return value instanceof byte[] ? value : ((ByteBuffer) value).array();
+ case ARRAY:
+ case ITERABLE:
+ Schema.FieldType elementType =
checkStateNotNull(type.getCollectionElementType());
+ List<@Nullable Object> list = new ArrayList<>();
+ for (@Nullable Object element : (Iterable<?>) value) {
+ list.add(toBeamNestedValue(elementType, element, fieldName));
+ }
+ return list;
+ case MAP:
+ Schema.FieldType keyType = checkStateNotNull(type.getMapKeyType());
+ Schema.FieldType valueType = checkStateNotNull(type.getMapValueType());
+ Map<@Nullable Object, @Nullable Object> map = new HashMap<>();
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
+ map.put(
+ toBeamNestedValue(keyType, entry.getKey(), fieldName),
+ toBeamNestedValue(valueType, entry.getValue(), fieldName));
+ }
+ return map;
+ default:
+ return value;
Review Comment:
Thanks for the catch. Added some checks that would throw during conversion
instead. PTAL
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]