yirutang commented on code in PR #24145:
URL: https://github.com/apache/beam/pull/24145#discussion_r1067552860
##########
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java:
##########
@@ -1035,6 +1038,78 @@ public void testRejectUnknownNestedFieldF() throws
Exception {
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA);
thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class);
- TableRowToStorageApiProto.messageFromTableRow(schemaInformation,
descriptor, topRow, false);
+
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, topRow, false, false, null);
+ }
+
+ @Test
+ public void testIgnoreUnknownField() throws Exception {
+ TableRow row = new TableRow();
+ row.putAll(BASE_TABLE_ROW_NO_F);
+ row.set("unknown", "foobar");
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA_NO_F,
true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA_NO_F);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(1, ignored.size());
+ assertEquals("foobar", ignored.get("unknown"));
+ }
+
+ @Test
+ public void testIgnoreUnknownFieldF() throws Exception {
+ TableRow row = new TableRow();
+ List<TableCell> cells = Lists.newArrayList(BASE_TABLE_ROW.getF());
+ cells.add(new TableCell().setV("foobar"));
+ row.setF(cells);
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA, true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(BASE_TABLE_ROW.getF().size() + 1, ignored.getF().size());
+ assertEquals("foobar",
ignored.getF().get(BASE_TABLE_ROW.getF().size()).getV());
+ }
+
+ @Test
+ public void testIgnoreUnknownNestedField() throws Exception {
+ TableRow rowNoF = new TableRow();
+ rowNoF.putAll(BASE_TABLE_ROW_NO_F);
Review Comment:
Just try to understand the test, so Beam accepts two format of TableRow? One
is "F"->List of values and the other is List(field_name, field_value), and they
can be mixed up as nested field value to the same TableRow struct?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
