This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new cd42544842 Synchronize after merge: rows sent to the error hop also go
to the main output, fixes #9247 (#9250)
cd42544842 is described below
commit cd42544842e84488a7259a8f6b35cc31d447299e
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 4 17:09:36 2026 +0100
Synchronize after merge: rows sent to the error hop also go to the main
output, fixes #9247 (#9250)
---
.../SynchronizeAfterMerge.java | 9 +-
.../SynchronizeAfterMergeErrorRowTest.java | 174 +++++++++++++++++++++
.../SynchronizeAfterMergePostgresIT.java | 54 ++++++-
3 files changed, 225 insertions(+), 12 deletions(-)
diff --git
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
index ef204921b5..c900957760 100644
---
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
+++
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
@@ -486,6 +486,9 @@ public class SynchronizeAfterMerge
} else {
putError(data.inputRowMeta, row, 1, errorMessage, null, "SUYNC001");
}
+ } else {
+ // A row goes to the error hop or to the output, never to both.
+ putRow(data.outputRowMeta, row);
}
}
}
@@ -772,10 +775,8 @@ public class SynchronizeAfterMerge
}
try {
- lookupValues(nextRow); // add new values to the row in rowset[0].
- if (!data.batchMode) {
- putRow(data.outputRowMeta, nextRow); // copy row to output rowset(s)
- }
+ // Passes the row on to the output or the error hop, in batch mode once
it is committed.
+ lookupValues(nextRow);
if (checkFeedback(getLinesRead()) && isDetailed()) {
logDetailed(
diff --git
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
new file mode 100644
index 0000000000..bd0a1975ca
--- /dev/null
+++
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.synchronizeaftermerge;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+import java.sql.PreparedStatement;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.database.Database;
+import org.apache.hop.core.exception.HopDatabaseException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformPartitioningMeta;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Without batch updates, a row whose statement fails goes to the error hop
only. It used to be
+ * passed on to the output as well, so the transforms after it got rows that
were never written.
+ */
+class SynchronizeAfterMergeErrorRowTest {
+
+ private SynchronizeAfterMerge transform;
+ private SynchronizeAfterMergeData data;
+ private List<String> output;
+ private List<String> errors;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ SynchronizeAfterMergeMeta meta = mock(SynchronizeAfterMergeMeta.class);
+ TransformMeta transformMeta = mock(TransformMeta.class);
+ doReturn("transform").when(transformMeta).getName();
+ doReturn(mock(TransformPartitioningMeta.class))
+ .when(transformMeta)
+ .getTargetTransformPartitioningMeta();
+ doReturn(meta).when(transformMeta).getTransform();
+ doReturn(true).when(transformMeta).isDoingErrorHandling();
+
+ PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+ doReturn(transformMeta).when(pipelineMeta).findTransform(anyString());
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaString("op"));
+
+ data = new SynchronizeAfterMergeData();
+ data.inputRowMeta = rowMeta;
+ data.outputRowMeta = rowMeta;
+ data.insertRowMeta = new RowMeta();
+ data.insertRowMeta.addValueMeta(new ValueMetaString("name"));
+ data.indexOfOperationOrderField = 1;
+ data.valueIndexes = new int[] {0};
+ data.keyIndexes = new int[0];
+ data.keyIndexes2 = new int[0];
+ data.insertValue = "insert";
+ data.updateValue = "update";
+ data.deleteValue = "delete";
+ data.insertStatement = mock(PreparedStatement.class);
+ data.commitSize = 100;
+ data.batchMode = false;
+ data.useSavepoints = false;
+
+ // The INSERT fails for a row named "bad".
+ Database db = mock(Database.class);
+ String[] bound = new String[1];
+ doAnswer(
+ inv -> {
+ Object[] values = inv.getArgument(1);
+ bound[0] = (String) values[0];
+ return null;
+ })
+ .when(db)
+ .setValues(any(IRowMeta.class), any(Object[].class),
any(PreparedStatement.class));
+ doAnswer(
+ inv -> {
+ if ("bad".equals(bound[0])) {
+ throw new HopDatabaseException("value too long");
+ }
+ return true;
+ })
+ .when(db)
+ .insertRow(any(PreparedStatement.class), anyBoolean());
+ data.db = db;
+
+ output = new ArrayList<>();
+ errors = new ArrayList<>();
+
+ transform =
+ spy(
+ new SynchronizeAfterMerge(
+ transformMeta, meta, data, 0, pipelineMeta, spy(new
LocalPipelineEngine())));
+ transform.first = false;
+ doReturn(transformMeta).when(transform).getTransformMeta();
+ doReturn(false).when(transform).isRowLevel();
+ doReturn(rowMeta).when(transform).getInputRowMeta();
+ doAnswer(
+ inv -> {
+ output.add((String) ((Object[]) inv.getArgument(1))[0]);
+ return null;
+ })
+ .when(transform)
+ .putRow(any(IRowMeta.class), any());
+ doAnswer(
+ inv -> {
+ errors.add((String) ((Object[]) inv.getArgument(1))[0]);
+ return null;
+ })
+ .when(transform)
+ .putError(
+ any(IRowMeta.class),
+ any(),
+ anyLong(),
+ anyString(),
+ nullable(String.class),
+ anyString());
+ }
+
+ private void run(Object[]... rows) throws Exception {
+ for (Object[] row : rows) {
+ doReturn(row).when(transform).getRow();
+ assertTrue(transform.processRow());
+ }
+ }
+
+ @Test
+ void aFailedRowGoesToTheErrorHopOnly() throws Exception {
+ run(
+ new Object[] {"good1", "insert"},
+ new Object[] {"bad", "insert"},
+ new Object[] {"good2", "insert"});
+
+ assertEquals(List.of("good1", "good2"), output);
+ assertEquals(List.of("bad"), errors);
+ assertEquals(0, transform.getErrors());
+ }
+
+ @Test
+ void aSkippedRowStillGoesToTheOutput() throws Exception {
+ run(new Object[] {"other", "nothing to do"}, new Object[] {"good",
"insert"});
+
+ assertEquals(List.of("other", "good"), output);
+ assertTrue(errors.isEmpty());
+ }
+}
diff --git
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
index 66662c027b..57d7f163d6 100644
---
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
+++
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
@@ -25,7 +25,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Queue;
+import java.util.Set;
import java.util.StringJoiner;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.hop.core.HopClientEnvironment;
import org.apache.hop.core.database.Database;
@@ -188,9 +190,30 @@ class SynchronizeAfterMergePostgresIT {
return content.toString();
}
+ /**
+ * Every input row leaves the transform once: on the output or on the error
hop, never on both.
+ */
+ private static void assertEveryRowLeavesOnce(
+ int rows, List<Object[]> output, List<Object[]> errors) {
+ Set<Long> seen = new TreeSet<>();
+ List<Object[]> all = new ArrayList<>(output);
+ if (errors != null) {
+ all.addAll(errors);
+ }
+ for (Object[] row : all) {
+ assertTrue(seen.add((Long) row[0]), "row " + row[0] + " left the
transform twice");
+ }
+ assertEquals(rows, seen.size(), "rows left on neither stream");
+ }
+
/** A Synchronize after merge, with an error hop when errors isn't null. */
private static SynchronizeAfterMerge synchronize(
- String name, String table, int commitSize, Queue<Object[]> input,
List<Object[]> errors)
+ String name,
+ String table,
+ int commitSize,
+ Queue<Object[]> input,
+ List<Object[]> output,
+ List<Object[]> errors)
throws Exception {
IRowMeta inputRowMeta = inputRowMeta();
PipelineMeta pipelineMeta = new PipelineMeta();
@@ -250,8 +273,7 @@ class SynchronizeAfterMergePostgresIT {
@Override
public void putRow(IRowMeta rowMeta, Object[] row) {
- // Not counted: without batch updates processRow() also passes on
a row that went
- // to the error hop, which this test doesn't cover.
+ output.add(row);
}
@Override
@@ -285,9 +307,10 @@ class SynchronizeAfterMergePostgresIT {
@ValueSource(ints = {100, 1})
void onlyTheFailingRowsGoToTheErrorHop(int commitSize) throws Exception {
createTarget("sync_target", ROWS);
+ List<Object[]> output = new ArrayList<>();
List<Object[]> errors = new ArrayList<>();
SynchronizeAfterMerge transform =
- synchronize("sync", "sync_target", commitSize, rows(ROWS, "a", 97),
errors);
+ synchronize("sync", "sync_target", commitSize, rows(ROWS, "a", 97),
output, errors);
assertTrue(transform.init());
try {
@@ -300,6 +323,7 @@ class SynchronizeAfterMergePostgresIT {
assertEquals(0, transform.getErrors(), "the transform failed");
assertEquals(expectedErrors(ROWS, 97), errors.size());
+ assertEveryRowLeavesOnce(ROWS, output, errors);
assertEquals(expectedTable(ROWS, "a", 97), table("sync_target"));
}
@@ -314,14 +338,18 @@ class SynchronizeAfterMergePostgresIT {
createTarget("sync_tx_b", ROWS);
List<Object[]> errorsA = Collections.synchronizedList(new ArrayList<>());
List<Object[]> errorsB = Collections.synchronizedList(new ArrayList<>());
+ List<Object[]> outputA = Collections.synchronizedList(new ArrayList<>());
+ List<Object[]> outputB = Collections.synchronizedList(new ArrayList<>());
SharedConnectionRun run = new SharedConnectionRun();
- run.join(synchronize("sync A", "sync_tx_a", commitSize, rows(ROWS, "a",
97), errorsA));
- run.join(synchronize("sync B", "sync_tx_b", commitSize, rows(ROWS, "b",
89), errorsB));
+ run.join(synchronize("sync A", "sync_tx_a", commitSize, rows(ROWS, "a",
97), outputA, errorsA));
+ run.join(synchronize("sync B", "sync_tx_b", commitSize, rows(ROWS, "b",
89), outputB, errorsB));
assertTrue(run.run(), "the transaction was rolled back");
assertEquals(expectedErrors(ROWS, 97), errorsA.size());
assertEquals(expectedErrors(ROWS, 89), errorsB.size());
+ assertEveryRowLeavesOnce(ROWS, outputA, errorsA);
+ assertEveryRowLeavesOnce(ROWS, outputB, errorsB);
assertEquals(expectedTable(ROWS, "a", 97), table("sync_tx_a"));
assertEquals(expectedTable(ROWS, "b", 89), table("sync_tx_b"));
}
@@ -332,15 +360,25 @@ class SynchronizeAfterMergePostgresIT {
createTarget("sync_tx_err", ROWS);
createTarget("sync_tx_plain", ROWS);
List<Object[]> errors = Collections.synchronizedList(new ArrayList<>());
+ List<Object[]> output = Collections.synchronizedList(new ArrayList<>());
+ List<Object[]> plainOutput = Collections.synchronizedList(new
ArrayList<>());
SharedConnectionRun run = new SharedConnectionRun();
- run.join(synchronize("with error hop", "sync_tx_err", 100, rows(ROWS, "a",
13), errors));
+ run.join(
+ synchronize("with error hop", "sync_tx_err", 100, rows(ROWS, "a", 13),
output, errors));
run.join(
synchronize(
- "without error hop", "sync_tx_plain", 100, rows(ROWS, "p",
Integer.MAX_VALUE), null));
+ "without error hop",
+ "sync_tx_plain",
+ 100,
+ rows(ROWS, "p", Integer.MAX_VALUE),
+ plainOutput,
+ null));
assertTrue(run.run(), "the transaction was rolled back");
assertEquals(expectedErrors(ROWS, 13), errors.size());
+ assertEveryRowLeavesOnce(ROWS, output, errors);
+ assertEveryRowLeavesOnce(ROWS, plainOutput, null);
assertEquals(expectedTable(ROWS, "a", 13), table("sync_tx_err"));
assertEquals(expectedTable(ROWS, "p", Integer.MAX_VALUE),
table("sync_tx_plain"));
}