This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 0545dae22e Data set input performance improvements, fixes #9435 (#9436)
0545dae22e is described below

commit 0545dae22e7f08fd1e4f6ae00985b89ea04d3cd8
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 12 19:46:22 2026 +0100

    Data set input performance improvements, fixes #9435 (#9436)
---
 .../main/java/org/apache/hop/testing/DataSet.java  |  10 ++
 .../org/apache/hop/testing/DataSetCsvUtil.java     |  53 ++++--
 .../transforms/datasetinput/DataSetInput.java      |  12 +-
 .../hop/testing/DataSetCsvUtilReadRowsTest.java    | 171 +++++++++++++++++++
 .../transforms/datasetinput/DataSetInputTest.java  | 183 +++++++++++++++++++++
 5 files changed, 413 insertions(+), 16 deletions(-)

diff --git 
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java 
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
index 1feb07da99..bab3ea016e 100644
--- a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
+++ b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSet.java
@@ -199,6 +199,16 @@ public class DataSet extends HopMetadataBase implements 
Cloneable, IHopMetadata
     return DataSetCsvUtil.getAllRows(variables, this);
   }
 
+  /**
+   * Reads the rows of this data set one at a time, without loading them all.
+   *
+   * @param handler receives each row in file order; return false to stop 
reading
+   */
+  public void readRows(IVariables variables, DataSetCsvUtil.IRowHandler 
handler)
+      throws HopException {
+    DataSetCsvUtil.readRows(variables, this, handler);
+  }
+
   /**
    * Calculate the row metadata for the data set fields needed for the given 
location.
    *
diff --git 
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java 
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
index c0e5e1c05a..4a14b98e56 100644
--- 
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
+++ 
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/DataSetCsvUtil.java
@@ -81,20 +81,53 @@ public class DataSetCsvUtil {
     }
   }
 
+  /** Receives the rows of a data set one at a time, in file order. */
+  @FunctionalInterface
+  public interface IRowHandler {
+    /**
+     * @param row the next row, in the layout of {@link 
DataSet#getSetRowMeta()}
+     * @return true to go on reading, false to stop
+     */
+    boolean handleRow(Object[] row) throws HopException;
+  }
+
   public static final List<Object[]> getAllRows(IVariables variables, DataSet 
dataSet)
       throws HopException {
+    List<Object[]> rows = new ArrayList<>();
+    readRows(variables, dataSet, rows::add);
+    return rows;
+  }
+
+  /**
+   * Reads the rows of a data set and hands each one to the handler as soon as 
it is parsed, so a
+   * large data set never has to fit in memory. A data set without a file has 
no rows yet: the
+   * handler is not called.
+   *
+   * @param handler receives each row; return false to stop reading
+   */
+  public static final void readRows(IVariables variables, DataSet dataSet, 
IRowHandler handler)
+      throws HopException {
     IRowMeta setRowMeta = dataSet.getSetRowMeta();
     setValueFormats(setRowMeta);
     String dataSetFilename = dataSet.getActualDataSetFilename(variables);
-    List<Object[]> rows = new ArrayList<>();
-    final ValueMetaString constantValueMeta = new ValueMetaString("constant");
+
+    // One String meta per field, prepared once: it carries the field's mask 
and symbols for the
+    // conversion. The field metadata keeps its number and date formats from 
row to row.
+    int nrFields = setRowMeta.size();
+    IValueMeta[] valueMetas = new IValueMeta[nrFields];
+    IValueMeta[] stringMetas = new IValueMeta[nrFields];
+    for (int i = 0; i < nrFields; i++) {
+      valueMetas[i] = setRowMeta.getValueMeta(i);
+      stringMetas[i] = new ValueMetaString("constant");
+      stringMetas[i].setConversionMetadata(valueMetas[i]);
+    }
 
     try {
       FileObject file = HopVfs.getFileObject(dataSetFilename);
       if (!file.exists()) {
         // This is fine.  We haven't put rows in yet.
         //
-        return rows;
+        return;
       }
 
       try (Reader reader =
@@ -102,20 +135,18 @@ public class DataSetCsvUtil {
           CSVParser csvParser = new CSVParser(reader, 
getCsvFormat(setRowMeta)); ) {
         for (CSVRecord csvRecord : csvParser) {
           if (csvRecord.getRecordNumber() > 1) {
-            Object[] row = RowDataUtil.allocateRowData(setRowMeta.size());
-            for (int i = 0; i < setRowMeta.size(); i++) {
-              IValueMeta valueMeta = setRowMeta.getValueMeta(i).clone();
-              constantValueMeta.setConversionMetadata(valueMeta);
+            Object[] row = RowDataUtil.allocateRowData(nrFields);
+            for (int i = 0; i < nrFields; i++) {
               if (i < csvRecord.size()) {
-                String value = csvRecord.get(i);
-                row[i] = csvValueToField(valueMeta, constantValueMeta, value);
+                row[i] = csvValueToField(valueMetas[i], stringMetas[i], 
csvRecord.get(i));
               }
             }
-            rows.add(row);
+            if (!handler.handleRow(row)) {
+              return;
+            }
           }
         }
       }
-      return rows;
     } catch (Exception e) {
       throw new HopException(
           "Unable to get all rows for CSV data set '" + dataSet.getName() + 
"'", e);
diff --git 
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
 
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
index d033b9cc5f..1956194349 100644
--- 
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
+++ 
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/transforms/datasetinput/DataSetInput.java
@@ -17,7 +17,6 @@
 
 package org.apache.hop.testing.transforms.datasetinput;
 
-import java.util.List;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hop.core.exception.HopException;
 import org.apache.hop.core.row.RowMeta;
@@ -70,10 +69,13 @@ public class DataSetInput extends 
BaseTransform<DataSetInputMeta, DataSetInputDa
       data.outputRowMeta.addRowMeta(data.dataSet.getSetRowMeta());
     }
 
-    List<Object[]> rows = data.dataSet.getAllRows(this, getLogChannel());
-    for (Object[] row : rows) {
-      putRow(data.outputRowMeta, row);
-    }
+    // Write every row as soon as it is read: the data set is never held in 
memory as a whole.
+    data.dataSet.readRows(
+        this,
+        row -> {
+          putRow(data.outputRowMeta, row);
+          return !isStopped();
+        });
     setOutputDone();
 
     return false;
diff --git 
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
 
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
new file mode 100644
index 0000000000..0d356e4ce1
--- /dev/null
+++ 
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/DataSetCsvUtilReadRowsTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.testing;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.Variables;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/** Reading a data set one row at a time gives the rows that reading it whole 
gives. */
+class DataSetCsvUtilReadRowsTest {
+
+  @TempDir Path tempDir;
+
+  private Variables variables;
+
+  @BeforeAll
+  static void initHop() throws HopException {
+    HopEnvironment.init();
+  }
+
+  @BeforeEach
+  void setUp() {
+    variables = new Variables();
+    variables.setVariable(DataSet.VARIABLE_HOP_DATASETS_FOLDER, 
tempDir.toString());
+  }
+
+  @Test
+  void everyFieldTypeIsReadBackOverManyRows() throws Exception {
+    DataSet dataSet = dataSet("types");
+    Date first = new Date(1_700_000_000_000L);
+    Date second = new Date(1_700_000_123_456L);
+    Object[][] written = {
+      {1L, "Alice", 3.14, new BigDecimal("12345.678"), first, true, new byte[] 
{1, 2, (byte) 255}},
+      {2L, "Bob", -0.5, new BigDecimal("-1"), second, false, new byte[] {0}},
+      {3L, null, null, null, null, null, null},
+      {4L, "Carol, \"quoted\"", 1e-20, new BigDecimal("0.000001"), first, 
true, new byte[] {9}}
+    };
+    try (DataSetCsvWriter writer =
+        new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+      for (Object[] row : written) {
+        writer.writeRow(row);
+      }
+    }
+
+    List<Object[]> streamed = new ArrayList<>();
+    DataSetCsvUtil.readRows(variables, dataSet, streamed::add);
+    List<Object[]> loaded = DataSetCsvUtil.getAllRows(variables, dataSet);
+
+    assertEquals(written.length, streamed.size());
+    assertEquals(written.length, loaded.size());
+    for (int r = 0; r < written.length; r++) {
+      for (int f = 0; f < written[r].length; f++) {
+        Object expected = written[r][f];
+        Object actual = streamed.get(r)[f];
+        if (expected instanceof byte[] bytes) {
+          assertArrayEquals(bytes, (byte[]) actual);
+          assertArrayEquals(bytes, (byte[]) loaded.get(r)[f]);
+        } else if (expected instanceof BigDecimal number) {
+          assertEquals(0, number.compareTo((BigDecimal) actual), "row " + r + 
" field " + f);
+          assertEquals(actual, loaded.get(r)[f]);
+        } else {
+          if (expected == null && f == 1) {
+            // CSV has no null for text: an empty String field reads back as 
an empty string
+            expected = "";
+          }
+          assertEquals(expected, actual, "row " + r + " field " + f);
+          assertEquals(actual, loaded.get(r)[f]);
+        }
+      }
+    }
+  }
+
+  @Test
+  void readingStopsWhenTheHandlerSaysSo() throws Exception {
+    DataSet dataSet = dataSet("stop");
+    try (DataSetCsvWriter writer =
+        new DataSetCsvWriter(variables, dataSet, dataSet.getSetRowMeta())) {
+      for (long i = 1; i <= 10; i++) {
+        writer.writeRow(new Object[] {i, "row " + i, null, null, null, null, 
null});
+      }
+    }
+
+    List<Object[]> rows = new ArrayList<>();
+    DataSetCsvUtil.readRows(
+        variables,
+        dataSet,
+        row -> {
+          rows.add(row);
+          return rows.size() < 3;
+        });
+
+    assertEquals(3, rows.size());
+    assertEquals(3L, rows.get(2)[0]);
+  }
+
+  @Test
+  void aDataSetWithoutAFileHasNoRows() throws Exception {
+    DataSet dataSet = dataSet("missing");
+
+    List<Object[]> rows = new ArrayList<>();
+    DataSetCsvUtil.readRows(variables, dataSet, rows::add);
+
+    assertTrue(rows.isEmpty());
+    assertTrue(DataSetCsvUtil.getAllRows(variables, dataSet).isEmpty());
+  }
+
+  @Test
+  void aShortRecordLeavesTheMissingFieldsEmpty() throws Exception {
+    DataSet dataSet = dataSet("short");
+    Files.writeString(
+        tempDir.resolve("short.csv"),
+        "id,name,amount,total,created,active,payload\n7,Dave\n",
+        StandardCharsets.UTF_8);
+
+    List<Object[]> rows = DataSetCsvUtil.getAllRows(variables, dataSet);
+
+    assertEquals(1, rows.size());
+    assertEquals(7L, rows.get(0)[0]);
+    assertEquals("Dave", rows.get(0)[1]);
+    for (int f = 2; f < 7; f++) {
+      assertNull(rows.get(0)[f]);
+    }
+  }
+
+  private static DataSet dataSet(String name) {
+    DataSet dataSet = new DataSet();
+    dataSet.setName(name);
+    dataSet.setBaseFilename(name + ".csv");
+    dataSet.setFields(
+        List.of(
+            new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+            new DataSetField("name", IValueMeta.TYPE_STRING, -1, -1, "", ""),
+            new DataSetField("amount", IValueMeta.TYPE_NUMBER, -1, -1, "", ""),
+            new DataSetField("total", IValueMeta.TYPE_BIGNUMBER, -1, -1, "", 
""),
+            new DataSetField("created", IValueMeta.TYPE_DATE, -1, -1, "", ""),
+            new DataSetField("active", IValueMeta.TYPE_BOOLEAN, -1, -1, "", 
""),
+            new DataSetField("payload", IValueMeta.TYPE_BINARY, -1, -1, "", 
"")));
+    return dataSet;
+  }
+}
diff --git 
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
 
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
new file mode 100644
index 0000000000..1ca3d31f3b
--- /dev/null
+++ 
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/transforms/datasetinput/DataSetInputTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.testing.transforms.datasetinput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hop.core.BlockingRowSet;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.apache.hop.testing.DataSet;
+import org.apache.hop.testing.DataSetCsvWriter;
+import org.apache.hop.testing.DataSetField;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class DataSetInputTest {
+
+  @TempDir Path tempDir;
+
+  private TransformMockHelper<DataSetInputMeta, DataSetInputData> mockHelper;
+  private MemoryMetadataProvider provider;
+
+  @BeforeAll
+  static void initHop() throws HopException {
+    HopEnvironment.init();
+  }
+
+  @BeforeEach
+  void setUp() {
+    mockHelper =
+        new TransformMockHelper<>("Data set input", DataSetInputMeta.class, 
DataSetInputData.class);
+    when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(mockHelper.iLogChannel);
+    when(mockHelper.pipeline.isRunning()).thenReturn(true);
+    when(mockHelper.transformMeta.getCopies(any())).thenReturn(1);
+    when(mockHelper.transformMeta.isPartitioned()).thenReturn(false);
+    provider = new MemoryMetadataProvider();
+  }
+
+  @AfterEach
+  void tearDown() {
+    mockHelper.cleanUp();
+  }
+
+  @Test
+  void everyRowIsWrittenInFileOrder() throws Exception {
+    DataSet dataSet = save("orders");
+    write(dataSet, 5);
+
+    DataSetInput transform = createTransform("orders", false);
+    BlockingRowSet output = output(transform);
+    assertTrue(transform.init());
+    assertFalse(transform.processRow());
+
+    for (long i = 1; i <= 5; i++) {
+      Object[] row = output.getRowWait(1, TimeUnit.SECONDS);
+      assertEquals(i, row[0]);
+      assertEquals("row " + i, row[1]);
+    }
+    assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+    assertTrue(output.isDone());
+  }
+
+  @Test
+  void aDataSetWithoutAFileGivesNoRows() throws Exception {
+    save("empty");
+
+    DataSetInput transform = createTransform("empty", false);
+    BlockingRowSet output = output(transform);
+    assertTrue(transform.init());
+    assertFalse(transform.processRow());
+
+    assertNull(output.getRowWait(100, TimeUnit.MILLISECONDS));
+    assertTrue(output.isDone());
+  }
+
+  /** The rows are written while the file is read, so a stop ends the reading 
too. */
+  @Test
+  void readingStopsWhenThePipelineIsStopped() throws Exception {
+    DataSet dataSet = save("stopped");
+    write(dataSet, 50);
+
+    DataSetInput transform = createTransform("stopped", true);
+    BlockingRowSet output = output(transform);
+    assertTrue(transform.init());
+    assertFalse(transform.processRow());
+
+    int rows = 0;
+    while (output.getRowWait(100, TimeUnit.MILLISECONDS) != null) {
+      rows++;
+    }
+    assertEquals(3, rows);
+  }
+
+  private DataSet save(String name) throws HopException {
+    DataSet dataSet = new DataSet();
+    dataSet.setName(name);
+    dataSet.setFolderName(tempDir.toString());
+    dataSet.setBaseFilename(name + ".csv");
+    dataSet.setFields(
+        List.of(
+            new DataSetField("id", IValueMeta.TYPE_INTEGER, -1, 0, "", "0"),
+            new DataSetField("name", IValueMeta.TYPE_STRING, -1, -1, "", "")));
+    provider.getSerializer(DataSet.class).save(dataSet);
+    return dataSet;
+  }
+
+  private void write(DataSet dataSet, int rows) throws Exception {
+    try (DataSetCsvWriter writer =
+        new DataSetCsvWriter(mockHelper.pipeline, dataSet, 
dataSet.getSetRowMeta())) {
+      for (long i = 1; i <= rows; i++) {
+        writer.writeRow(new Object[] {i, "row " + i});
+      }
+    }
+  }
+
+  private static BlockingRowSet output(DataSetInput transform) {
+    BlockingRowSet rowSet = new BlockingRowSet(100);
+    rowSet.setThreadNameFromToCopy("Data set input", 0, "next", 0);
+    transform.addRowSetToOutputRowSets(rowSet);
+    return rowSet;
+  }
+
+  /**
+   * @param stopAfterThreeRows stop the transform once it has written its 
third row
+   */
+  private DataSetInput createTransform(String dataSetName, boolean 
stopAfterThreeRows) {
+    DataSetInputMeta meta = new DataSetInputMeta();
+    meta.setDataSetName(dataSetName);
+    when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+    DataSetInput transform =
+        new DataSetInput(
+            mockHelper.transformMeta,
+            meta,
+            new DataSetInputData(),
+            0,
+            mockHelper.pipelineMeta,
+            mockHelper.pipeline) {
+          private int written;
+
+          @Override
+          public void putRow(IRowMeta rowMeta, Object[] row) throws 
HopTransformException {
+            super.putRow(rowMeta, row);
+            if (stopAfterThreeRows && ++written == 3) {
+              setStopped(true);
+            }
+          }
+        };
+    transform.setMetadataProvider(provider);
+    return transform;
+  }
+}

Reply via email to