This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 44b5dac407 fix Unique rows - error handling replaces fields with the
duplicate count (#6061)
44b5dac407 is described below
commit 44b5dac4070ec5b1b9e604d8aa8e22efaad09d72
Author: lance <[email protected]>
AuthorDate: Tue Dec 2 21:09:06 2025 +0800
fix Unique rows - error handling replaces fields with the duplicate count
(#6061)
Signed-off-by: lance <[email protected]>
---
.../pipeline/transforms/uniquerows/UniqueRows.java | 34 ++++++++---
.../transforms/uniquerows/UniqueRowsTest.java | 67 ++++++++++++++++++++++
2 files changed, 92 insertions(+), 9 deletions(-)
diff --git
a/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRows.java
b/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRows.java
index 82fa7221ac..7433ca996c 100644
---
a/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRows.java
+++
b/plugins/transforms/uniquerows/src/main/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRows.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.uniquerows;
import java.util.List;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowDataUtil;
import org.apache.hop.core.util.Utils;
@@ -123,15 +124,7 @@ public class UniqueRows extends
BaseTransform<UniqueRowsMeta, UniqueRowsData> {
data.previous = data.inputRowMeta.cloneRow(r);
data.counter = 1;
} else {
- if (data.sendDuplicateRows && !first) {
- putError(
- data.outputRowMeta,
- RowDataUtil.addValueData(r, data.outputRowMeta.size() - 1,
data.counter),
- 1,
- data.realErrorDescription,
- Utils.isEmpty(data.compareFields) ? null : data.compareFields,
- "UNR001");
- }
+ handleDuplicateErrorRow(r);
data.counter++;
}
@@ -142,6 +135,29 @@ public class UniqueRows extends
BaseTransform<UniqueRowsMeta, UniqueRowsData> {
return true;
}
+ /**
+ * Handles sending a duplicate row to the error stream.
+ *
+ * @param r input row
+ * @throws HopTransformException If routing the row to the error stream
fails.
+ */
+ private void handleDuplicateErrorRow(Object[] r) throws
HopTransformException {
+ if (data.sendDuplicateRows && !first) {
+ Object[] errRow =
+ meta.isCountRows()
+ ? RowDataUtil.addValueData(r, data.outputRowMeta.size() - 1,
data.counter)
+ : r;
+
+ putError(
+ data.outputRowMeta,
+ errRow,
+ 1,
+ data.realErrorDescription,
+ Utils.isEmpty(data.compareFields) ? null : data.compareFields,
+ "UNR001");
+ }
+ }
+
private Object[] addCounter(IRowMeta outputRowMeta, Object[] r, long count) {
if (meta.isCountRows()) {
return RowDataUtil.addValueData(r, outputRowMeta.size() - 1,
Long.valueOf(count));
diff --git
a/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsTest.java
b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsTest.java
index 625cb0dee7..cebbe6867f 100644
---
a/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsTest.java
+++
b/plugins/transforms/uniquerows/src/test/java/org/apache/hop/pipeline/transforms/uniquerows/UniqueRowsTest.java
@@ -21,9 +21,14 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -34,9 +39,11 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBoolean;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transform.TransformErrorMeta;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -327,4 +334,64 @@ class UniqueRowsTest {
// Verify that the previous row was output
verify(uniqueRows).putRow(any(IRowMeta.class), any(Object[].class));
}
+
+ @Test
+ void testDuplicateRowTriggersErrorHandling() throws HopException {
+ List<UniqueField> compareFields = new ArrayList<>();
+ compareFields.add(new UniqueField("name", false));
+
+
when(transformMockHelper.iTransformMeta.getCompareFields()).thenReturn(compareFields);
+ when(transformMockHelper.iTransformMeta.isCountRows()).thenReturn(false);
+ when(transformMockHelper.transformMeta.getTransformErrorMeta())
+ .thenReturn(mock(TransformErrorMeta.class));
+
when(transformMockHelper.transformMeta.getTransformErrorMeta().getErrorRowMeta(any()))
+ .thenReturn(mock(IRowMeta.class));
+
+ UniqueRows uniqueRows =
+ spy(
+ new UniqueRows(
+ transformMockHelper.transformMeta,
+ transformMockHelper.iTransformMeta,
+ transformMockHelper.iTransformData,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline));
+
+ assertTrue(uniqueRows.init());
+
+ // Setup input row meta
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("name"));
+ // The last field is Boolean to simulate a non-integer type.
+ inputRowMeta.addValueMeta(new ValueMetaBoolean("success"));
+ uniqueRows.setInputRowMeta(inputRowMeta);
+
+ // Set up previous row in data
+ transformMockHelper.iTransformData.previous = new Object[] {"Alice", true};
+ transformMockHelper.iTransformData.outputRowMeta = inputRowMeta;
+ transformMockHelper.iTransformData.counter = 2;
+ transformMockHelper.iTransformData.sendDuplicateRows = true;
+
+ doReturn(new Object[] {"lance", true})
+ .doReturn(new Object[] {"lance", false})
+ .doReturn(new Object[] {"lance", false})
+ .doReturn(null)
+ .when(uniqueRows)
+ .getRow();
+
+ doNothing()
+ .when(uniqueRows)
+ .putError(
+ any(IRowMeta.class), any(Object[].class), anyLong(), anyString(),
any(), anyString());
+
+ boolean result = uniqueRows.processRow();
+ assertTrue(result);
+
+ uniqueRows.processRow();
+ uniqueRows.processRow();
+
+ // verify putError triggered for duplicate
+ verify(uniqueRows, times(2))
+ .putError(any(IRowMeta.class), any(Object[].class), eq(1L), any(),
any(), eq("UNR001"));
+ }
}
