reuvenlax commented on code in PR #24145:
URL: https://github.com/apache/beam/pull/24145#discussion_r1081865565
##########
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowToStorageApiProtoTest.java:
##########
@@ -1035,6 +1038,78 @@ public void testRejectUnknownNestedFieldF() throws
Exception {
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA);
thrown.expect(TableRowToStorageApiProto.SchemaConversionException.class);
- TableRowToStorageApiProto.messageFromTableRow(schemaInformation,
descriptor, topRow, false);
+
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, topRow, false, false, null);
+ }
+
+ @Test
+ public void testIgnoreUnknownField() throws Exception {
+ TableRow row = new TableRow();
+ row.putAll(BASE_TABLE_ROW_NO_F);
+ row.set("unknown", "foobar");
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA_NO_F,
true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA_NO_F);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(1, ignored.size());
+ assertEquals("foobar", ignored.get("unknown"));
+ }
+
+ @Test
+ public void testIgnoreUnknownFieldF() throws Exception {
+ TableRow row = new TableRow();
+ List<TableCell> cells = Lists.newArrayList(BASE_TABLE_ROW.getF());
+ cells.add(new TableCell().setV("foobar"));
+ row.setF(cells);
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(BASE_TABLE_SCHEMA, true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(BASE_TABLE_SCHEMA);
+
+ TableRow ignored = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, row, true, false, ignored);
+ assertEquals(BASE_TABLE_ROW.getF().size() + 1, ignored.getF().size());
+ assertEquals("foobar",
ignored.getF().get(BASE_TABLE_ROW.getF().size()).getV());
+ }
+
+ @Test
+ public void testIgnoreUnknownNestedField() throws Exception {
+ TableRow rowNoF = new TableRow();
+ rowNoF.putAll(BASE_TABLE_ROW_NO_F);
+ rowNoF.set("unknown", "foobar");
+ TableRow rowWithF = new TableRow();
+ List<TableCell> cells = Lists.newArrayList(BASE_TABLE_ROW.getF());
+ cells.add(new TableCell().setV("foobar"));
+ rowWithF.setF(cells);
+ TableRow topRow =
+ new TableRow()
+ .set("nestedValueNoF1", rowNoF)
+ .set("nestedValue1", rowWithF)
+ .set("unknowntop", "foobar");
+
+ Descriptor descriptor =
+
TableRowToStorageApiProto.getDescriptorFromTableSchema(NESTED_TABLE_SCHEMA,
true);
+ TableRowToStorageApiProto.SchemaInformation schemaInformation =
+
TableRowToStorageApiProto.SchemaInformation.fromTableSchema(NESTED_TABLE_SCHEMA);
+
+ TableRow unknown = new TableRow();
+ TableRowToStorageApiProto.messageFromTableRow(
+ schemaInformation, descriptor, topRow, true, false, unknown);
+ assertEquals(3, unknown.size());
+ assertEquals("foobar", unknown.get("unknowntop"));
+ assertEquals(1, ((TableRow) unknown.get("nestedvalue1")).size());
+ assertEquals(1, ((TableRow) unknown.get("nestedvaluenof1")).size());
+ assertEquals(
+ "foobar",
+ ((TableRow)
unknown.get("nestedvalue1")).getF().get(BASE_TABLE_ROW.getF().size()).getV());
Review Comment:
This is an unordered map, so there should be no offsets.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
