This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new e6dda0598a PgVector Upsert: schema DDL changes commit mode on a shared 
connection, fixes #9249 (#9252)
e6dda0598a is described below

commit e6dda0598ad3e883501b384b981cec6536fb04b7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Nov 4 17:29:02 2026 +0100

    PgVector Upsert: schema DDL changes commit mode on a shared connection, 
fixes #9249 (#9252)
---
 .../pgvector/transforms/upsert/PgVectorUpsert.java | 14 +++-
 .../upsert/PgVectorUpsertPostgresIT.java           | 93 +++++++++++++++++++++-
 .../transforms/upsert/PgVectorUpsertTest.java      | 21 +++++
 3 files changed, 123 insertions(+), 5 deletions(-)

diff --git 
a/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
 
b/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
index 593f3fd249..6f6aebc654 100644
--- 
a/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
+++ 
b/plugins/tech/pgvector/src/main/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsert.java
@@ -317,12 +317,20 @@ public class PgVectorUpsert extends 
BaseTransform<PgVectorUpsertMeta, PgVectorUp
    * <p>The DDL runs with autocommit on. Inside the data transaction a table 
or index created here
    * would be undone by the rollback of a single failed row, and an index 
would hold its locks for
    * the length of the load.
+   *
+   * <p>Not on a connection shared by a transactional pipeline: switching it 
to autocommit commits
+   * the open transaction, with the rows the other transforms wrote so far, 
and a later rollback of
+   * the pipeline can no longer undo them. There the DDL runs in the 
pipeline's transaction (it is
+   * transactional in PostgreSQL), and the commit mode is the pipeline's to 
keep.
    */
   void prepareSchemaAndCommitMode(String schemaName, String tableName) throws 
HopException {
+    boolean sharedConnection = 
!Utils.isEmpty(data.database.getConnectionGroup());
     // The HNSW index and the mapped-column ALTERs are independent options, so 
they have to run on
     // an existing table too, not only when this transform creates one.
     if (meta.isCreateTableIfMissing() || meta.isCreateHnswIndex() || 
hasColumnMappings()) {
-      data.database.setCommit(0);
+      if (!sharedConnection) {
+        data.database.setCommit(0);
+      }
       VectorDistanceMetric indexMetric =
           meta.getIndexMetric() != null ? meta.getIndexMetric() : 
VectorDistanceMetric.COSINE;
       PgVectorDatabase.ensureSchema(
@@ -333,7 +341,9 @@ public class PgVectorUpsert extends 
BaseTransform<PgVectorUpsertMeta, PgVectorUp
           indexMetric,
           Const.toInt(resolve(meta.getEmbeddingDimensions()), 768));
     }
-    data.database.setCommit(data.commitSize);
+    if (!sharedConnection) {
+      data.database.setCommit(data.commitSize);
+    }
   }
 
   private void openDatabase() throws HopException {
diff --git 
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
 
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
index aa60ca1f71..477b89eb82 100644
--- 
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
+++ 
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertPostgresIT.java
@@ -26,6 +26,11 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
 import org.apache.hop.core.HopClientEnvironment;
 import org.apache.hop.core.database.Database;
 import org.apache.hop.core.database.DatabaseMeta;
@@ -151,6 +156,21 @@ class PgVectorUpsertPostgresIT {
   private static PgVectorUpsert upsert(
       String name, String table, int commitSize, Queue<Object[]> input, 
List<Object[]> errors)
       throws Exception {
+    return upsert(name, table, commitSize, input::poll, errors, false);
+  }
+
+  /**
+   * An upsert into table, with an error hop when errors isn't null, that 
creates the table and an
+   * HNSW index when createSchema is set.
+   */
+  private static PgVectorUpsert upsert(
+      String name,
+      String table,
+      int commitSize,
+      Supplier<Object[]> input,
+      List<Object[]> errors,
+      boolean createSchema)
+      throws Exception {
     IRowMeta inputRowMeta = inputRowMeta();
     PipelineMeta pipelineMeta = new PipelineMeta();
     // Not named metadataProvider: inside the transform that name is 
BaseTransform's own field.
@@ -163,8 +183,8 @@ class PgVectorUpsertPostgresIT {
     meta.setSchemaName("public");
     meta.setTableName(table);
     meta.setEmbeddingDimensions("2");
-    meta.setCreateTableIfMissing(false);
-    meta.setCreateHnswIndex(false);
+    meta.setCreateTableIfMissing(createSchema);
+    meta.setCreateHnswIndex(createSchema);
     meta.setCommitSize(Integer.toString(commitSize));
     TransformMeta transformMeta = new TransformMeta(name, meta);
     pipelineMeta.addTransform(transformMeta);
@@ -189,7 +209,7 @@ class PgVectorUpsertPostgresIT {
 
           @Override
           public Object[] getRow() {
-            Object[] row = input.poll();
+            Object[] row = input.get();
             return row == null ? null : row.clone();
           }
 
@@ -283,4 +303,71 @@ class PgVectorUpsertPostgresIT {
     assertEquals(ROWS - ROWS / 13, count("SELECT count(*) FROM vec_tx_err"));
     assertEquals(ROWS, count("SELECT count(*) FROM vec_tx_plain"));
   }
+
+  /**
+   * In a transactional pipeline the schema DDL runs in the pipeline's 
transaction. It used to
+   * switch the shared connection to autocommit, which commits the open 
transaction: the rows the
+   * other transforms had written so far were committed halfway through the 
run, so a later rollback
+   * of the pipeline could no longer undo them.
+   */
+  @Test
+  void theSchemaDdlDoesNotCommitTheTransactionOfAPipeline() throws Exception {
+    createTable("vec_tx_writer");
+    execute("DROP TABLE IF EXISTS vec_tx_ddl");
+    CountDownLatch writerWrote = new CountDownLatch(1);
+    CountDownLatch schemaCreated = new CountDownLatch(1);
+    AtomicLong committedHalfway = new AtomicLong(-1);
+
+    // The writer batches 20 rows into the transaction (commit size 10), then 
waits for the schema.
+    AtomicInteger written = new AtomicInteger();
+    Supplier<Object[]> writerInput =
+        () -> {
+          int chunk = written.incrementAndGet();
+          if (chunk <= 20) {
+            return new Object[] {"doc", (long) chunk, "w", "[1,2]"};
+          }
+          if (chunk == 21) {
+            writerWrote.countDown();
+            await(schemaCreated);
+            try {
+              committedHalfway.set(count("SELECT count(*) FROM 
vec_tx_writer"));
+            } catch (Exception e) {
+              throw new IllegalStateException(e);
+            }
+          }
+          return null;
+        };
+
+    // Its first row creates the table and index; asking for the second means 
that is done.
+    AtomicInteger asked = new AtomicInteger();
+    Supplier<Object[]> schemaInput =
+        () -> {
+          if (asked.incrementAndGet() == 1) {
+            await(writerWrote);
+            return new Object[] {"doc", 1L, "s", "[1,2]"};
+          }
+          schemaCreated.countDown();
+          return null;
+        };
+
+    SharedConnectionRun run = new SharedConnectionRun().timeoutSeconds(60);
+    run.join(upsert("writer", "vec_tx_writer", 10, writerInput, null, false));
+    run.join(upsert("with schema", "vec_tx_ddl", 10, schemaInput, null, true));
+    assertTrue(run.run(), "the transaction was rolled back");
+
+    assertEquals(0, committedHalfway.get(), "the rows of the writer were 
committed halfway");
+    assertEquals(20, count("SELECT count(*) FROM vec_tx_writer"));
+    assertEquals(1, count("SELECT count(*) FROM vec_tx_ddl"));
+  }
+
+  private static void await(CountDownLatch latch) {
+    try {
+      if (!latch.await(30, TimeUnit.SECONDS)) {
+        throw new IllegalStateException("timed out");
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IllegalStateException(e);
+    }
+  }
 }
diff --git 
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
 
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
index ec9849cb0d..7d8a4a7857 100644
--- 
a/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
+++ 
b/plugins/tech/pgvector/src/test/java/org/apache/hop/pgvector/transforms/upsert/PgVectorUpsertTest.java
@@ -306,6 +306,27 @@ class PgVectorUpsertTest {
     inOrder.verify(database).setCommit(500);
   }
 
+  @Test
+  void leavesTheCommitModeOfASharedConnectionAlone() throws Exception {
+    // Switching a connection shared by a transactional pipeline to autocommit 
commits the rows the
+    // other transforms wrote so far.
+    PgVectorUpsertMeta meta = newMeta();
+    meta.setCreateTableIfMissing(true);
+    meta.setCreateHnswIndex(true);
+    meta.setEmbeddingDimensions("768");
+
+    PgVectorUpsertData data = newData(meta, mock(PreparedStatement.class));
+    data.commitSize = 500;
+    when(database.getConnectionGroup()).thenReturn("pipeline-transaction");
+    PgVectorUpsert transform =
+        new PgVectorUpsert(
+            helper.transformMeta, meta, data, 0, helper.pipelineMeta, 
helper.pipeline);
+    transform.prepareSchemaAndCommitMode("public", "chunks");
+
+    verify(database, atLeastOnce()).execStatement(anyString());
+    verify(database, never()).setCommit(anyInt());
+  }
+
   @Test
   void refusesToDeleteADocumentOnceTheTrackedSetIsFull() throws Exception {
     // Evicting an entry would let a document be deleted twice, the second 
time taking the chunks

Reply via email to