rzepinskip commented on code in PR #11857: URL: https://github.com/apache/nifi/pull/11857#discussion_r4492807803
########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_VALUE); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapperWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadataWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffsetWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } Review Comment: Right, we can bring them back if we decide to differentiate behavior by Output Strategy. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } Review Comment: Right, we can bring them back if we decide to differentiate behavior by Output Strategy. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -177,6 +184,55 @@ public void testVerifyFailed() throws InitializationException { assertNotNull(firstResult.getExplanation()); } + @Test + public void testVerifySkipsTombstoneAndParsesFollowingRecord() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + final RecordReader recordReader = mock(RecordReader.class); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())).thenAnswer(invocation -> { + final long length = invocation.getArgument(2); + if (length == 0L) { + throw new IOException("empty value"); + } + + return recordReader; + }); + + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone, dataRecord)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.SUCCESSFUL, parseResult.getOutcome()); + assertEquals("Successfully parsed 1 records (skipped 1 tombstones)", parseResult.getExplanation()); + } + + @Test + public void testVerifySkipsWhenSampleContainsOnlyTombstones() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.SKIPPED, parseResult.getOutcome()); + assertEquals("No records were parsed (skipped 1 tombstones)", parseResult.getExplanation()); + verify(readerFactory, never()).createRecordReader(any(), any(), anyLong(), any()); + } + + @Test + public void testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())).thenThrow(new IOException("empty value")); + + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone, dataRecord)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.FAILED, parseResult.getOutcome()); + assertEquals("Failed to parse Record number 2: java.io.IOException: empty value", parseResult.getExplanation()); + verify(readerFactory).createRecordReader(any(), any(), anyLong(), any()); + } Review Comment: I'd rather keep these as two tests. They share the same two records, but they check different verify() outcomes: a successful parse after skipping the tombstone, vs a failure whose record number still counts the skipped tombstone. Keeping them separate allows to make this outcomes explicit in test names. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -365,6 +421,32 @@ public void testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); } + private RecordReaderFactory recordReaderFactory() throws InitializationException { + final RecordReaderFactory readerFactory = mock(RecordReaderFactory.class); + when(readerFactory.getIdentifier()).thenReturn(READER_ID); + runner.addControllerService(READER_ID, readerFactory); + runner.enableControllerService(readerFactory); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.RECORD_READER, READER_ID); + return readerFactory; + } + + private List<ConfigVerificationResult> verifyRecords() throws InitializationException { + when(kafkaConsumerService.getPartitionStates()).thenReturn(List.of(new PartitionState(TEST_TOPIC_NAME, FIRST_PARTITION))); + setConnectionService(); + when(kafkaConnectionService.getConsumerService(any())).thenReturn(kafkaConsumerService); + runner.setProperty(TOPICS, TEST_TOPIC_NAME); + runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID); + return processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap()); + } + + private static ConfigVerificationResult parseResult(final List<ConfigVerificationResult> results) { Review Comment: Ok, fixed. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/record/ByteRecord.java: ########## @@ -45,6 +46,20 @@ public ByteRecord( final byte[] key, final byte[] value, final long bundledCount + ) { + this(topic, partition, offset, timestamp, headers, key, value, bundledCount, false); + } + + public ByteRecord( + final String topic, + final int partition, + final long offset, + final long timestamp, + final List<RecordHeader> headers, + final byte[] key, + final byte[] value, + final long bundledCount, + final boolean tombstone Review Comment: The 8-arg constructor is the previous public API on nifi-kafka-service-api. Production still uses it in ByteRecordBundler (DEMARCATOR strategy), and defaulting tombstone to false is correct there (bundled records are not Kafka tombstones). Kafka consumer code uses the new 9-arg constructor with the real flag (true or false depending on `org.apache.kafka.clients.consumer.ConsumerRecord.value()`). I could remove existing 8-arg constructor and use new 9-arg one for ByteRecordBundler as well but wouldn't that be an API break (that would mean? ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_VALUE); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapperWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadataWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffsetWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + private void setOutputStrategy(final OutputStrategy outputStrategy) { + runner.setProperty(ConsumeKafka.OUTPUT_STRATEGY, outputStrategy); + } + + private void produceJsonRecords() throws ExecutionException, InterruptedException { Review Comment: Ok, fixed. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverterTest.java: ########## @@ -176,4 +179,40 @@ void testParseFailureIncrementsParseErrorCounterImmediately() throws Exception { KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(FIRST_PARTITION))), eq(CommitTiming.NOW)); } + + @Test + void testTombstoneSkipsReaderAndEmptyValueStillFailsParsing() throws Exception { + final OffsetTracker tombstoneOffsetTracker = new OffsetTracker(); + final RecordReaderFactory readerFactory = mock(RecordReaderFactory.class); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())) + .thenThrow(new MalformedRecordException("empty value")); + + final FlowFile flowFile = new MockFlowFile(1); + when(session.create()).thenReturn(flowFile); + when(session.putAllAttributes(any(FlowFile.class), any())).thenReturn(flowFile); + when(session.write(any(FlowFile.class), any(OutputStreamCallback.class))).thenReturn(flowFile); + + final RecordStreamKafkaMessageConverter converter = new RecordStreamKafkaMessageConverter( + readerFactory, + writerFactory, + value -> new String(value, StandardCharsets.UTF_8), + Pattern.compile(".*"), + KeyEncoding.UTF8, + true, + tombstoneOffsetTracker, + logger, + BROKER_URI, + new CreateNewFlowFileGrouping(writerFactory, logger, BROKER_URI, true) + ); + + final ByteRecord tombstone = new ByteRecord(FIRST_TOPIC, FIRST_PARTITION, 4L, 1000L, List.of(), null, new byte[0], 1L, true); + final ByteRecord emptyValue = new ByteRecord(FIRST_TOPIC, FIRST_PARTITION, 5L, 1000L, List.of(), null, new byte[0], 1L); + converter.toFlowFiles(session, List.of(tombstone, emptyValue).iterator()); + + verify(readerFactory, times(1)).createRecordReader(any(), any(), anyLong(), any()); + verify(session).transfer(flowFile, ConsumeKafka.PARSE_FAILURE); + verify(session, times(2)).adjustCounter(eq("Records Received from " + FIRST_TOPIC), eq(1L), eq(false)); + verify(session).adjustCounter(eq("Tombstones Skipped for " + FIRST_TOPIC), eq(1L), eq(false)); + assertEquals(2L, tombstoneOffsetTracker.getRecordCounts().get(FIRST_TOPIC)); + } Review Comment: Ok, fixed. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -365,6 +421,32 @@ public void testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); } + private RecordReaderFactory recordReaderFactory() throws InitializationException { Review Comment: Ok, fixed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
