chamikaramj commented on code in PR #40998:
URL: https://github.com/apache/beam/pull/40998#discussion_r4580731077


##########
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java:
##########
@@ -684,24 +692,62 @@ private static void addIcebergValue(
                 field.getType().getRowSchema(),
                 "Corrupted schema: Row type did not have associated nested 
schema.");
         if (icebergValue instanceof Record) {
-          rowBuilder.addValue(icebergRecordToBeamRow(nestedSchema, (Record) 
icebergValue));
+          values.add(icebergRecordToBeamRow(nestedSchema, (Record) 
icebergValue));
         } else if (icebergValue instanceof StructLike) {
-          rowBuilder.addValue(structToRow(nestedSchema, (StructLike) 
icebergValue));
+          values.add(structToRow(nestedSchema, (StructLike) icebergValue));
         } else {
           throw new UnsupportedOperationException(
               "Unsupported row type: " + icebergValue.getClass());
         }
         break;
       case LOGICAL_TYPE:
-        rowBuilder.addValue(getLogicalTypeValue(icebergValue, 
field.getType()));
+        values.add(getLogicalTypeValue(icebergValue, field.getType()));
         break;
       default:
         throw new UnsupportedOperationException(
             "Unsupported Beam type: " + field.getType().getTypeName());
     }
   }
 
-  private static DateTime getBeamDateTimeValue(Object icebergValue) {
+  /**
+   * Copies a list or map, or one of its elements, the way {@link 
Row.Builder#build()} would. A null
+   * fails where the Beam type is not nullable, and {@code binary}, which 
Iceberg reads as a {@link
+   * ByteBuffer}, becomes byte[].
+   */
+  private static @Nullable Object toBeamNestedValue(
+      Schema.FieldType type, @Nullable Object value, String fieldName) {
+    if (value == null) {
+      checkArgument(
+          type.getNullable(), "Received null value for %s in field '%s'.", 
type, fieldName);
+      return null;
+    }
+    switch (type.getTypeName()) {
+      case BYTES:
+        return value instanceof byte[] ? value : ((ByteBuffer) value).array();
+      case ARRAY:
+      case ITERABLE:
+        Schema.FieldType elementType = 
checkStateNotNull(type.getCollectionElementType());
+        List<@Nullable Object> list = new ArrayList<>();
+        for (@Nullable Object element : (Iterable<?>) value) {
+          list.add(toBeamNestedValue(elementType, element, fieldName));
+        }
+        return list;
+      case MAP:
+        Schema.FieldType keyType = checkStateNotNull(type.getMapKeyType());
+        Schema.FieldType valueType = checkStateNotNull(type.getMapValueType());
+        Map<@Nullable Object, @Nullable Object> map = new HashMap<>();
+        for (Map.Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
+          map.put(
+              toBeamNestedValue(keyType, entry.getKey(), fieldName),
+              toBeamNestedValue(valueType, entry.getValue(), fieldName));
+        }
+        return map;
+      default:
+        return value;

Review Comment:
   Probably also check if we need additional tests if the previous build tests 
are not covering some of these scenarios anymore.



##########
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java:
##########
@@ -684,24 +692,62 @@ private static void addIcebergValue(
                 field.getType().getRowSchema(),
                 "Corrupted schema: Row type did not have associated nested 
schema.");
         if (icebergValue instanceof Record) {
-          rowBuilder.addValue(icebergRecordToBeamRow(nestedSchema, (Record) 
icebergValue));
+          values.add(icebergRecordToBeamRow(nestedSchema, (Record) 
icebergValue));
         } else if (icebergValue instanceof StructLike) {
-          rowBuilder.addValue(structToRow(nestedSchema, (StructLike) 
icebergValue));
+          values.add(structToRow(nestedSchema, (StructLike) icebergValue));
         } else {
           throw new UnsupportedOperationException(
               "Unsupported row type: " + icebergValue.getClass());
         }
         break;
       case LOGICAL_TYPE:
-        rowBuilder.addValue(getLogicalTypeValue(icebergValue, 
field.getType()));
+        values.add(getLogicalTypeValue(icebergValue, field.getType()));
         break;
       default:
         throw new UnsupportedOperationException(
             "Unsupported Beam type: " + field.getType().getTypeName());
     }
   }
 
-  private static DateTime getBeamDateTimeValue(Object icebergValue) {
+  /**
+   * Copies a list or map, or one of its elements, the way {@link 
Row.Builder#build()} would. A null
+   * fails where the Beam type is not nullable, and {@code binary}, which 
Iceberg reads as a {@link
+   * ByteBuffer}, becomes byte[].
+   */
+  private static @Nullable Object toBeamNestedValue(
+      Schema.FieldType type, @Nullable Object value, String fieldName) {
+    if (value == null) {
+      checkArgument(
+          type.getNullable(), "Received null value for %s in field '%s'.", 
type, fieldName);
+      return null;
+    }
+    switch (type.getTypeName()) {
+      case BYTES:
+        return value instanceof byte[] ? value : ((ByteBuffer) value).array();
+      case ARRAY:
+      case ITERABLE:
+        Schema.FieldType elementType = 
checkStateNotNull(type.getCollectionElementType());
+        List<@Nullable Object> list = new ArrayList<>();
+        for (@Nullable Object element : (Iterable<?>) value) {
+          list.add(toBeamNestedValue(elementType, element, fieldName));
+        }
+        return list;
+      case MAP:
+        Schema.FieldType keyType = checkStateNotNull(type.getMapKeyType());
+        Schema.FieldType valueType = checkStateNotNull(type.getMapValueType());
+        Map<@Nullable Object, @Nullable Object> map = new HashMap<>();
+        for (Map.Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
+          map.put(
+              toBeamNestedValue(keyType, entry.getKey(), fieldName),
+              toBeamNestedValue(valueType, entry.getValue(), fieldName));
+        }
+        return map;
+      default:
+        return value;

Review Comment:
   Could not converting nested DATETIME / LOGICAL_TYPE / ROW values result in a 
early error for incorrect values being differed till encoding here ?
   
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to