yirutang commented on code in PR #24145:
URL: https://github.com/apache/beam/pull/24145#discussion_r1067561105
##########
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java:
##########
@@ -1035,6 +1038,78 @@ public void testRejectUnknownNestedFieldF() throws
Exception {
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA);
thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class);
- TableRowToStorageApiProto.messageFromTableRow(schemaInformation,
descriptor, topRow, false);
+
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, topRow, false, false, null);
+ }
+
+ @Test
+ public void testIgnoreUnknownField() throws Exception {
+ TableRow row = new TableRow();
+ row.putAll(BASE_TABLE_ROW_NO_F);
+ row.set("unknown", "foobar");
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA_NO_F,
true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA_NO_F);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(1, ignored.size());
+ assertEquals("foobar", ignored.get("unknown"));
+ }
+
+ @Test
+ public void testIgnoreUnknownFieldF() throws Exception {
+ TableRow row = new TableRow();
+ List<TableCell> cells = Lists.newArrayList(BASE_TABLE_ROW.getF());
+ cells.add(new TableCell().setV("foobar"));
+ row.setF(cells);
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA, true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(BASE_TABLE_ROW.getF().size() + 1, ignored.getF().size());
+ assertEquals("foobar",
ignored.getF().get(BASE_TABLE_ROW.getF().size()).getV());
+ }
+
+ @Test
+ public void testIgnoreUnknownNestedField() throws Exception {
+ TableRow rowNoF = new TableRow();
+ rowNoF.putAll(BASE_TABLE_ROW_NO_F);
+ rowNoF.set("unknown", "foobar");
+ TableRow rowWithF = new TableRow();
+ List<TableCell> cells = Lists.newArrayList(BASE_TABLE_ROW.getF());
+ cells.add(new TableCell().setV("foobar"));
+ rowWithF.setF(cells);
+ TableRow topRow =
+ new TableRow()
+ .set("nestedValueNoF1", rowNoF)
+ .set("nestedValue1", rowWithF)
+ .set("unknowntop", "foobar");
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA,
true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA);
+
+ TableRow unknown = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, topRow, true, false, unknown);
+ assertEquals(3, unknown.size());
+ assertEquals("foobar", unknown.get("unknowntop"));
+ assertEquals(1, ((TableRow) unknown.get("nestedvalue1")).size());
+ assertEquals(1, ((TableRow) unknown.get("nestedvaluenof1")).size());
+ assertEquals(
+ "foobar",
+ ((TableRow)
unknown.get("nestedvalue1")).getF().get(BASE_TABLE_ROW.getF().size()).getV());
Review Comment:
In this case, the unknown value is at the last offset, is there elements
ahead of it? What if the unknown field is in the middle of the array list?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
