markap14 commented on code in PR #11857:
URL: https://github.com/apache/nifi/pull/11857#discussion_r4485652852


##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }

Review Comment:
   [Claude Opus-5.5] These four JSON reader tests pass on `main` without this 
change. As your table in the earlier comment shows, with the JSON reader a null 
value already produced no FlowFile on either `success` or `parse.failure`. So 
they don't protect against the bug this PR fixes, and each one costs a full 
round trip to Kafka. I'd suggest removing them.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, 
InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapperWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadataWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffsetWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }

Review Comment:
   [Claude Opus-5.5] The Avro tests are the ones that actually reproduce the 
bug, but the four of them differ only by Output Strategy. Tombstones are 
skipped in `AbstractRecordStreamKafkaMessageConverter.toFlowFiles` before any 
code specific to an Output Strategy runs, so the strategy doesn't change the 
result. A single Avro test (or at most `USE_VALUE` and `USE_WRAPPER`) would 
give the same protection with far fewer Kafka round trips.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -177,6 +184,55 @@ public void testVerifyFailed() throws 
InitializationException {
         assertNotNull(firstResult.getExplanation());
     }
 
+    @Test
+    public void testVerifySkipsTombstoneAndParsesFollowingRecord() throws 
Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        final RecordReader recordReader = mock(RecordReader.class);
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), 
any())).thenAnswer(invocation -> {
+            final long length = invocation.getArgument(2);
+            if (length == 0L) {
+                throw new IOException("empty value");
+            }
+
+            return recordReader;
+        });
+
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone,
 dataRecord));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.SUCCESSFUL, parseResult.getOutcome());
+        assertEquals("Successfully parsed 1 records (skipped 1 tombstones)", 
parseResult.getExplanation());
+    }
+
+    @Test
+    public void testVerifySkipsWhenSampleContainsOnlyTombstones() throws 
Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.SKIPPED, parseResult.getOutcome());
+        assertEquals("No records were parsed (skipped 1 tombstones)", 
parseResult.getExplanation());
+        verify(readerFactory, never()).createRecordReader(any(), any(), 
anyLong(), any());
+    }
+
+    @Test
+    public void testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure() 
throws Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), 
any())).thenThrow(new IOException("empty value"));
+
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone,
 dataRecord));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.FAILED, parseResult.getOutcome());
+        assertEquals("Failed to parse Record number 2: java.io.IOException: 
empty value", parseResult.getExplanation());
+        verify(readerFactory).createRecordReader(any(), any(), anyLong(), 
any());
+    }

Review Comment:
   [Claude Opus-5.5] `testVerifySkipsTombstoneAndParsesFollowingRecord` and 
`testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure` use the same two 
records and differ only in how the mock reader behaves. They could be combined 
into one test: run verify and check the successful result, then change the 
reader to throw and run verify again to check the failure message and record 
number.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverterTest.java:
##########
@@ -176,4 +179,40 @@ void 
testParseFailureIncrementsParseErrorCounterImmediately() throws Exception {
                         KafkaFlowFileAttribute.KAFKA_PARTITION, 
Integer.toString(FIRST_PARTITION))),
                 eq(CommitTiming.NOW));
     }
+
+    @Test
+    void testTombstoneSkipsReaderAndEmptyValueStillFailsParsing() throws 
Exception {
+        final OffsetTracker tombstoneOffsetTracker = new OffsetTracker();
+        final RecordReaderFactory readerFactory = 
mock(RecordReaderFactory.class);
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), any()))
+                .thenThrow(new MalformedRecordException("empty value"));
+
+        final FlowFile flowFile = new MockFlowFile(1);
+        when(session.create()).thenReturn(flowFile);
+        when(session.putAllAttributes(any(FlowFile.class), 
any())).thenReturn(flowFile);
+        when(session.write(any(FlowFile.class), 
any(OutputStreamCallback.class))).thenReturn(flowFile);
+
+        final RecordStreamKafkaMessageConverter converter = new 
RecordStreamKafkaMessageConverter(
+                readerFactory,
+                writerFactory,
+                value -> new String(value, StandardCharsets.UTF_8),
+                Pattern.compile(".*"),
+                KeyEncoding.UTF8,
+                true,
+                tombstoneOffsetTracker,
+                logger,
+                BROKER_URI,
+                new CreateNewFlowFileGrouping(writerFactory, logger, 
BROKER_URI, true)
+        );
+
+        final ByteRecord tombstone = new ByteRecord(FIRST_TOPIC, 
FIRST_PARTITION, 4L, 1000L, List.of(), null, new byte[0], 1L, true);
+        final ByteRecord emptyValue = new ByteRecord(FIRST_TOPIC, 
FIRST_PARTITION, 5L, 1000L, List.of(), null, new byte[0], 1L);
+        converter.toFlowFiles(session, List.of(tombstone, 
emptyValue).iterator());
+
+        verify(readerFactory, times(1)).createRecordReader(any(), any(), 
anyLong(), any());
+        verify(session).transfer(flowFile, ConsumeKafka.PARSE_FAILURE);
+        verify(session, times(2)).adjustCounter(eq("Records Received from " + 
FIRST_TOPIC), eq(1L), eq(false));
+        verify(session).adjustCounter(eq("Tombstones Skipped for " + 
FIRST_TOPIC), eq(1L), eq(false));
+        assertEquals(2L, 
tombstoneOffsetTracker.getRecordCounts().get(FIRST_TOPIC));
+    }

Review Comment:
   [Claude Opus-5.5] This repeats nearly all of the setup in 
`testParseFailureIncrementsParseErrorCounterImmediately` just above. Could we 
add the tombstone record to that existing test instead, along with the counter 
and offset assertions from here? That keeps the number of tests the same.
   
   Also, the separate `tombstoneOffsetTracker` isn't needed: JUnit creates a 
new instance of the test class for every test, so the `offsetTracker` field is 
already fresh. And `times(1)` is Mockito's default, so `verify(readerFactory)` 
is enough.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -365,6 +421,32 @@ public void 
testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa
         runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
     }
 
+    private RecordReaderFactory recordReaderFactory() throws 
InitializationException {

Review Comment:
   [Claude Opus-5.5] Small naming suggestion: method names read best when they 
say what the method does. Besides creating the mock, this adds and enables the 
service and switches the processor to RECORD mode. Something like 
`configureRecordReader()` would make that clearer where it's called.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -365,6 +421,32 @@ public void 
testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa
         runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
     }
 
+    private RecordReaderFactory recordReaderFactory() throws 
InitializationException {
+        final RecordReaderFactory readerFactory = 
mock(RecordReaderFactory.class);
+        when(readerFactory.getIdentifier()).thenReturn(READER_ID);
+        runner.addControllerService(READER_ID, readerFactory);
+        runner.enableControllerService(readerFactory);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.RECORD_READER, READER_ID);
+        return readerFactory;
+    }
+
+    private List<ConfigVerificationResult> verifyRecords() throws 
InitializationException {
+        when(kafkaConsumerService.getPartitionStates()).thenReturn(List.of(new 
PartitionState(TEST_TOPIC_NAME, FIRST_PARTITION)));
+        setConnectionService();
+        
when(kafkaConnectionService.getConsumerService(any())).thenReturn(kafkaConsumerService);
+        runner.setProperty(TOPICS, TEST_TOPIC_NAME);
+        runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
+        return processor.verify(runner.getProcessContext(), 
runner.getLogger(), Collections.emptyMap());
+    }
+
+    private static ConfigVerificationResult parseResult(final 
List<ConfigVerificationResult> results) {

Review Comment:
   [Claude Opus-5.5] Same idea here: `findParseResult()` would say what this 
method does.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, 
InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapperWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadataWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffsetWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    private void setOutputStrategy(final OutputStrategy outputStrategy) {
+        runner.setProperty(ConsumeKafka.OUTPUT_STRATEGY, outputStrategy);
+    }
+
+    private void produceJsonRecords() throws ExecutionException, 
InterruptedException {

Review Comment:
   [Claude Opus-5.5] Naming suggestion: `produceJsonRecords()` and 
`produceAvroRecords()` also start the processor and run it until output 
appears, which the names don't suggest. Something like 
`produceAndConsumeJsonRecords()` / `produceAndConsumeAvroRecords()` would be 
clearer. Similarly, `avroRecord()` below would read better as 
`createAvroRecord()`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to