This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new cd42544842 Synchronize after merge: rows sent to the error hop also go 
to the main output, fixes #9247 (#9250)
cd42544842 is described below

commit cd42544842e84488a7259a8f6b35cc31d447299e
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 4 17:09:36 2026 +0100

    Synchronize after merge: rows sent to the error hop also go to the main 
output, fixes #9247 (#9250)
---
 .../SynchronizeAfterMerge.java                     |   9 +-
 .../SynchronizeAfterMergeErrorRowTest.java         | 174 +++++++++++++++++++++
 .../SynchronizeAfterMergePostgresIT.java           |  54 ++++++-
 3 files changed, 225 insertions(+), 12 deletions(-)

diff --git 
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
 
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
index ef204921b5..c900957760 100644
--- 
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
+++ 
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java
@@ -486,6 +486,9 @@ public class SynchronizeAfterMerge
         } else {
           putError(data.inputRowMeta, row, 1, errorMessage, null, "SUYNC001");
         }
+      } else {
+        // A row goes to the error hop or to the output, never to both.
+        putRow(data.outputRowMeta, row);
       }
     }
   }
@@ -772,10 +775,8 @@ public class SynchronizeAfterMerge
     }
 
     try {
-      lookupValues(nextRow); // add new values to the row in rowset[0].
-      if (!data.batchMode) {
-        putRow(data.outputRowMeta, nextRow); // copy row to output rowset(s)
-      }
+      // Passes the row on to the output or the error hop, in batch mode once 
it is committed.
+      lookupValues(nextRow);
 
       if (checkFeedback(getLinesRead()) && isDetailed()) {
         logDetailed(
diff --git 
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
 
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
new file mode 100644
index 0000000000..bd0a1975ca
--- /dev/null
+++ 
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeErrorRowTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.synchronizeaftermerge;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+import java.sql.PreparedStatement;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.database.Database;
+import org.apache.hop.core.exception.HopDatabaseException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transform.TransformPartitioningMeta;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Without batch updates, a row whose statement fails goes to the error hop 
only. It used to be
+ * passed on to the output as well, so the transforms after it got rows that 
were never written.
+ */
+class SynchronizeAfterMergeErrorRowTest {
+
+  private SynchronizeAfterMerge transform;
+  private SynchronizeAfterMergeData data;
+  private List<String> output;
+  private List<String> errors;
+
+  @BeforeEach
+  void setUp() throws Exception {
+    SynchronizeAfterMergeMeta meta = mock(SynchronizeAfterMergeMeta.class);
+    TransformMeta transformMeta = mock(TransformMeta.class);
+    doReturn("transform").when(transformMeta).getName();
+    doReturn(mock(TransformPartitioningMeta.class))
+        .when(transformMeta)
+        .getTargetTransformPartitioningMeta();
+    doReturn(meta).when(transformMeta).getTransform();
+    doReturn(true).when(transformMeta).isDoingErrorHandling();
+
+    PipelineMeta pipelineMeta = mock(PipelineMeta.class);
+    doReturn(transformMeta).when(pipelineMeta).findTransform(anyString());
+
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaString("name"));
+    rowMeta.addValueMeta(new ValueMetaString("op"));
+
+    data = new SynchronizeAfterMergeData();
+    data.inputRowMeta = rowMeta;
+    data.outputRowMeta = rowMeta;
+    data.insertRowMeta = new RowMeta();
+    data.insertRowMeta.addValueMeta(new ValueMetaString("name"));
+    data.indexOfOperationOrderField = 1;
+    data.valueIndexes = new int[] {0};
+    data.keyIndexes = new int[0];
+    data.keyIndexes2 = new int[0];
+    data.insertValue = "insert";
+    data.updateValue = "update";
+    data.deleteValue = "delete";
+    data.insertStatement = mock(PreparedStatement.class);
+    data.commitSize = 100;
+    data.batchMode = false;
+    data.useSavepoints = false;
+
+    // The INSERT fails for a row named "bad".
+    Database db = mock(Database.class);
+    String[] bound = new String[1];
+    doAnswer(
+            inv -> {
+              Object[] values = inv.getArgument(1);
+              bound[0] = (String) values[0];
+              return null;
+            })
+        .when(db)
+        .setValues(any(IRowMeta.class), any(Object[].class), 
any(PreparedStatement.class));
+    doAnswer(
+            inv -> {
+              if ("bad".equals(bound[0])) {
+                throw new HopDatabaseException("value too long");
+              }
+              return true;
+            })
+        .when(db)
+        .insertRow(any(PreparedStatement.class), anyBoolean());
+    data.db = db;
+
+    output = new ArrayList<>();
+    errors = new ArrayList<>();
+
+    transform =
+        spy(
+            new SynchronizeAfterMerge(
+                transformMeta, meta, data, 0, pipelineMeta, spy(new 
LocalPipelineEngine())));
+    transform.first = false;
+    doReturn(transformMeta).when(transform).getTransformMeta();
+    doReturn(false).when(transform).isRowLevel();
+    doReturn(rowMeta).when(transform).getInputRowMeta();
+    doAnswer(
+            inv -> {
+              output.add((String) ((Object[]) inv.getArgument(1))[0]);
+              return null;
+            })
+        .when(transform)
+        .putRow(any(IRowMeta.class), any());
+    doAnswer(
+            inv -> {
+              errors.add((String) ((Object[]) inv.getArgument(1))[0]);
+              return null;
+            })
+        .when(transform)
+        .putError(
+            any(IRowMeta.class),
+            any(),
+            anyLong(),
+            anyString(),
+            nullable(String.class),
+            anyString());
+  }
+
+  private void run(Object[]... rows) throws Exception {
+    for (Object[] row : rows) {
+      doReturn(row).when(transform).getRow();
+      assertTrue(transform.processRow());
+    }
+  }
+
+  @Test
+  void aFailedRowGoesToTheErrorHopOnly() throws Exception {
+    run(
+        new Object[] {"good1", "insert"},
+        new Object[] {"bad", "insert"},
+        new Object[] {"good2", "insert"});
+
+    assertEquals(List.of("good1", "good2"), output);
+    assertEquals(List.of("bad"), errors);
+    assertEquals(0, transform.getErrors());
+  }
+
+  @Test
+  void aSkippedRowStillGoesToTheOutput() throws Exception {
+    run(new Object[] {"other", "nothing to do"}, new Object[] {"good", 
"insert"});
+
+    assertEquals(List.of("other", "good"), output);
+    assertTrue(errors.isEmpty());
+  }
+}
diff --git 
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
 
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
index 66662c027b..57d7f163d6 100644
--- 
a/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
+++ 
b/plugins/transforms/synchronizeaftermerge/src/test/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergePostgresIT.java
@@ -25,7 +25,9 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Queue;
+import java.util.Set;
 import java.util.StringJoiner;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import org.apache.hop.core.HopClientEnvironment;
 import org.apache.hop.core.database.Database;
@@ -188,9 +190,30 @@ class SynchronizeAfterMergePostgresIT {
     return content.toString();
   }
 
+  /**
+   * Every input row leaves the transform once: on the output or on the error 
hop, never on both.
+   */
+  private static void assertEveryRowLeavesOnce(
+      int rows, List<Object[]> output, List<Object[]> errors) {
+    Set<Long> seen = new TreeSet<>();
+    List<Object[]> all = new ArrayList<>(output);
+    if (errors != null) {
+      all.addAll(errors);
+    }
+    for (Object[] row : all) {
+      assertTrue(seen.add((Long) row[0]), "row " + row[0] + " left the 
transform twice");
+    }
+    assertEquals(rows, seen.size(), "rows left on neither stream");
+  }
+
   /** A Synchronize after merge, with an error hop when errors isn't null. */
   private static SynchronizeAfterMerge synchronize(
-      String name, String table, int commitSize, Queue<Object[]> input, 
List<Object[]> errors)
+      String name,
+      String table,
+      int commitSize,
+      Queue<Object[]> input,
+      List<Object[]> output,
+      List<Object[]> errors)
       throws Exception {
     IRowMeta inputRowMeta = inputRowMeta();
     PipelineMeta pipelineMeta = new PipelineMeta();
@@ -250,8 +273,7 @@ class SynchronizeAfterMergePostgresIT {
 
           @Override
           public void putRow(IRowMeta rowMeta, Object[] row) {
-            // Not counted: without batch updates processRow() also passes on 
a row that went
-            // to the error hop, which this test doesn't cover.
+            output.add(row);
           }
 
           @Override
@@ -285,9 +307,10 @@ class SynchronizeAfterMergePostgresIT {
   @ValueSource(ints = {100, 1})
   void onlyTheFailingRowsGoToTheErrorHop(int commitSize) throws Exception {
     createTarget("sync_target", ROWS);
+    List<Object[]> output = new ArrayList<>();
     List<Object[]> errors = new ArrayList<>();
     SynchronizeAfterMerge transform =
-        synchronize("sync", "sync_target", commitSize, rows(ROWS, "a", 97), 
errors);
+        synchronize("sync", "sync_target", commitSize, rows(ROWS, "a", 97), 
output, errors);
 
     assertTrue(transform.init());
     try {
@@ -300,6 +323,7 @@ class SynchronizeAfterMergePostgresIT {
 
     assertEquals(0, transform.getErrors(), "the transform failed");
     assertEquals(expectedErrors(ROWS, 97), errors.size());
+    assertEveryRowLeavesOnce(ROWS, output, errors);
     assertEquals(expectedTable(ROWS, "a", 97), table("sync_target"));
   }
 
@@ -314,14 +338,18 @@ class SynchronizeAfterMergePostgresIT {
     createTarget("sync_tx_b", ROWS);
     List<Object[]> errorsA = Collections.synchronizedList(new ArrayList<>());
     List<Object[]> errorsB = Collections.synchronizedList(new ArrayList<>());
+    List<Object[]> outputA = Collections.synchronizedList(new ArrayList<>());
+    List<Object[]> outputB = Collections.synchronizedList(new ArrayList<>());
 
     SharedConnectionRun run = new SharedConnectionRun();
-    run.join(synchronize("sync A", "sync_tx_a", commitSize, rows(ROWS, "a", 
97), errorsA));
-    run.join(synchronize("sync B", "sync_tx_b", commitSize, rows(ROWS, "b", 
89), errorsB));
+    run.join(synchronize("sync A", "sync_tx_a", commitSize, rows(ROWS, "a", 
97), outputA, errorsA));
+    run.join(synchronize("sync B", "sync_tx_b", commitSize, rows(ROWS, "b", 
89), outputB, errorsB));
     assertTrue(run.run(), "the transaction was rolled back");
 
     assertEquals(expectedErrors(ROWS, 97), errorsA.size());
     assertEquals(expectedErrors(ROWS, 89), errorsB.size());
+    assertEveryRowLeavesOnce(ROWS, outputA, errorsA);
+    assertEveryRowLeavesOnce(ROWS, outputB, errorsB);
     assertEquals(expectedTable(ROWS, "a", 97), table("sync_tx_a"));
     assertEquals(expectedTable(ROWS, "b", 89), table("sync_tx_b"));
   }
@@ -332,15 +360,25 @@ class SynchronizeAfterMergePostgresIT {
     createTarget("sync_tx_err", ROWS);
     createTarget("sync_tx_plain", ROWS);
     List<Object[]> errors = Collections.synchronizedList(new ArrayList<>());
+    List<Object[]> output = Collections.synchronizedList(new ArrayList<>());
+    List<Object[]> plainOutput = Collections.synchronizedList(new 
ArrayList<>());
 
     SharedConnectionRun run = new SharedConnectionRun();
-    run.join(synchronize("with error hop", "sync_tx_err", 100, rows(ROWS, "a", 
13), errors));
+    run.join(
+        synchronize("with error hop", "sync_tx_err", 100, rows(ROWS, "a", 13), 
output, errors));
     run.join(
         synchronize(
-            "without error hop", "sync_tx_plain", 100, rows(ROWS, "p", 
Integer.MAX_VALUE), null));
+            "without error hop",
+            "sync_tx_plain",
+            100,
+            rows(ROWS, "p", Integer.MAX_VALUE),
+            plainOutput,
+            null));
     assertTrue(run.run(), "the transaction was rolled back");
 
     assertEquals(expectedErrors(ROWS, 13), errors.size());
+    assertEveryRowLeavesOnce(ROWS, output, errors);
+    assertEveryRowLeavesOnce(ROWS, plainOutput, null);
     assertEquals(expectedTable(ROWS, "a", 13), table("sync_tx_err"));
     assertEquals(expectedTable(ROWS, "p", Integer.MAX_VALUE), 
table("sync_tx_plain"));
   }

Reply via email to