mark-bathori commented on code in PR #7421:
URL: https://github.com/apache/nifi/pull/7421#discussion_r1273229002
##########
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestPutIcebergWithHiveCatalog.java:
##########
@@ -139,16 +139,15 @@ private void initCatalog(PartitionSpec spec, String
fileFormat) throws Initializ
runner.setProperty(PutIceberg.CATALOG, "catalog-service");
}
- @ParameterizedTest
- @ValueSource(strings = {"avro"})
- public void onTriggerPartitioned(String fileFormat) throws Exception {
+ @Test
+ public void onTriggerPartitioned() throws Exception {
PartitionSpec spec = PartitionSpec.builderFor(USER_SCHEMA)
.bucket("department", 3)
.build();
runner = TestRunners.newTestRunner(processor);
initRecordReader();
- initCatalog(spec, fileFormat);
+ initCatalog(spec, FileFormat.AVRO.name());
Review Comment:
I meant you should change the initCatalog method:
`initCatalog(PartitionSpec spec, FileFormat fileFormat)`
And call the `name()` method in that function so you don't need to call it
at every method call.
##########
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java:
##########
@@ -466,8 +511,52 @@ public void testPrimitives(FileFormat format) throws
IOException {
assertEquals(LocalDateTime.of(2017, 4, 4, 14, 20, 33, 789000000),
resultRecord.get(12, LocalDateTime.class));
assertEquals(Integer.valueOf(10), resultRecord.get(14, Integer.class));
- if (format.equals(PARQUET)) {
- assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0}, resultRecord.get(13, byte[].class));
+ assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0}, resultRecord.get(13, byte[].class));
+ }
+
+ @DisabledOnOs(WINDOWS)
+ @ParameterizedTest
+ @EnumSource(value = FileFormat.class, names = {"AVRO", "ORC", "PARQUET"})
+ public void testPrimitivesIgnoreMissingFields(FileFormat format) throws
IOException {
Review Comment:
This test is still failing on AVRO and PARQUET if the Iceberg schema fields
are set to required from optional.
##########
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/main/java/org/apache/nifi/processors/iceberg/converter/IcebergRecordConverter.java:
##########
@@ -123,8 +135,12 @@ private static class IcebergSchemaVisitor extends
SchemaWithPartnerVisitor<DataT
// set NiFi schema field names (sourceFieldName) in the data
converters
for (DataConverter<?, ?> converter : converters) {
final Optional<String> mappedFieldName =
recordType.getNameMapping(converter.getTargetFieldName());
- final Optional<RecordField> recordField =
recordSchema.getField(mappedFieldName.get());
- converter.setSourceFieldName(recordField.get().getFieldName());
+ if (mappedFieldName.isEmpty()) {
+
converter.setSourceFieldName(converter.getTargetFieldName());
+ } else {
+ final Optional<RecordField> recordField =
recordSchema.getField(mappedFieldName.get());
+
converter.setSourceFieldName(recordField.get().getFieldName());
+ }
Review Comment:
This still needs to be addressed.
##########
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/TestIcebergRecordConverter.java:
##########
@@ -466,8 +511,52 @@ public void testPrimitives(FileFormat format) throws
IOException {
assertEquals(LocalDateTime.of(2017, 4, 4, 14, 20, 33, 789000000),
resultRecord.get(12, LocalDateTime.class));
assertEquals(Integer.valueOf(10), resultRecord.get(14, Integer.class));
- if (format.equals(PARQUET)) {
- assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0}, resultRecord.get(13, byte[].class));
+ assertArrayEquals(new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0}, resultRecord.get(13, byte[].class));
+ }
+
+ @DisabledOnOs(WINDOWS)
+ @ParameterizedTest
+ @EnumSource(value = FileFormat.class, names = {"AVRO", "ORC", "PARQUET"})
+ public void testPrimitivesIgnoreMissingFields(FileFormat format) throws
IOException {
Review Comment:
This test is still failing on AVRO and PARQUET if the Iceberg schema fields
are set to required from optional.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
