This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 859a6274de Make the Hop GUI row samplers cheaper per row, fixes #9307
(#9308)
859a6274de is described below
commit 859a6274de819f3d6dd847cf7f428a99c24c165f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 07:23:14 2026 +0100
Make the Hop GUI row samplers cheaper per row, fixes #9307 (#9308)
---
.../hopgui/file/pipeline/HopGuiPipelineGraph.java | 9 +-
.../file/shared/PipelineRowSamplerHelper.java | 248 +++++++++++++++------
.../file/shared/PipelineRowSamplerHelperTest.java | 134 +++++++++++
3 files changed, 320 insertions(+), 71 deletions(-)
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
index adeab15e1f..6080e9943f 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
@@ -35,6 +35,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.Timer;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@@ -6575,8 +6576,8 @@ public class HopGuiPipelineGraph extends
HopGuiAbstractGraph
private void addRowsSamplerToPipeline(IPipelineEngine<PipelineMeta>
pipeline) {
try {
- outputRowsMap = new HashMap<>();
- outputHopRowsMap = new HashMap<>();
+ outputRowsMap = new ConcurrentHashMap<>();
+ outputHopRowsMap = new ConcurrentHashMap<>();
PipelineRowSamplerHelper.addRowSamplersToPipeline(pipeline,
outputRowsMap, outputHopRowsMap);
} catch (Exception e) {
// Ignore: run config not local or sample type not set
@@ -6592,8 +6593,8 @@ public class HopGuiPipelineGraph extends
HopGuiAbstractGraph
if (pipeline == null) {
return;
}
- outputRowsMap = new HashMap<>();
- outputHopRowsMap = new HashMap<>();
+ outputRowsMap = new ConcurrentHashMap<>();
+ outputHopRowsMap = new ConcurrentHashMap<>();
PipelineRowSamplerHelper.addRowSamplersToPipeline(pipeline, outputRowsMap,
outputHopRowsMap);
}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
index f4412eedd7..ba9cb007a4 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelper.java
@@ -17,13 +17,12 @@
package org.apache.hop.ui.hopgui.file.shared;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.IRowSet;
@@ -48,6 +47,11 @@ import org.apache.hop.pipeline.transform.RowToAdapter;
* <p>Supports both transform-level samples ({@code IRowListener} / {@code
putRow}) and hop-level
* samples ({@code IRowToListener} / {@code putRowTo}) for target hops such as
Filter and
* Switch/Case.
+ *
+ * <p>The listeners run for every row the sampled transform writes, so they do
as little as possible
+ * per row: each listener resolves its buffer once, and a row is only cloned
when it is kept. First
+ * stops cloning once the buffer is full, Random clones only the rows that
land in the reservoir,
+ * and Last keeps a fixed ring of rows instead of a linked list.
*/
public final class PipelineRowSamplerHelper {
@@ -135,11 +139,6 @@ public final class PipelineRowSamplerHelper {
if (meta == null) {
return;
}
- // Created once per pipeline start and shared by all the row listeners
below
- @SuppressWarnings("java:S2119")
- final Random random = new Random();
- // Per-key counters for Random sampling (transform name or hop key)
- final Map<String, AtomicInteger> rowCounters = new ConcurrentHashMap<>();
for (final String transformName : meta.getTransformNames()) {
IEngineComponent component = pipeline.findComponent(transformName, 0);
if (component == null) {
@@ -149,18 +148,18 @@ public final class PipelineRowSamplerHelper {
// Transform-level samples (main putRow path)
component.addRowListener(
new RowAdapter() {
+ private RowSampler sampler;
+
@Override
public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
throws HopTransformException {
- applySample(
- targetMap,
- transformName,
- rowMeta,
- row,
- sampleType,
- sampleSize,
- random,
- rowCounters);
+ RowSampler s = sampler;
+ if (s == null) {
+ s = createSampler(sampleType, sampleSize, rowMeta);
+ targetMap.put(transformName, s.getRowBuffer());
+ sampler = s;
+ }
+ s.sample(rowMeta, row);
}
});
@@ -168,72 +167,187 @@ public final class PipelineRowSamplerHelper {
if (hopTargetMap != null) {
component.addRowToListener(
new RowToAdapter() {
+ // One sampler per target row set, so the hop key is built once
per hop, not per row
+ private final Map<IRowSet, RowSampler> samplers = new
ConcurrentHashMap<>();
+
@Override
public void rowWrittenTo(IRowMeta rowMeta, Object[] row, IRowSet
rowSet)
throws HopTransformException {
- String key = hopKey(rowSet);
- if (StringUtils.isEmpty(key) || key.equals(HOP_KEY_SEPARATOR))
{
+ if (rowSet == null) {
return;
}
- applySample(
- hopTargetMap, key, rowMeta, row, sampleType, sampleSize,
random, rowCounters);
+ RowSampler s = samplers.get(rowSet);
+ if (s == null) {
+ String key = hopKey(rowSet);
+ if (StringUtils.isEmpty(key) ||
key.equals(HOP_KEY_SEPARATOR)) {
+ return;
+ }
+ s = createSampler(sampleType, sampleSize, rowMeta);
+ hopTargetMap.put(key, s.getRowBuffer());
+ samplers.put(rowSet, s);
+ }
+ s.sample(rowMeta, row);
}
});
}
}
}
- /** Apply First/Last/Random sampling into the map. */
- private static void applySample(
- Map<String, RowBuffer> targetMap,
- String key,
- IRowMeta rowMeta,
- Object[] row,
- SampleType sampleType,
- int sampleSize,
- Random random,
- Map<String, AtomicInteger> rowCounters)
- throws HopTransformException {
- RowBuffer rowBuffer = targetMap.get(key);
- if (rowBuffer == null) {
- rowBuffer = new RowBuffer(rowMeta);
- if (sampleType == SampleType.Last) {
- rowBuffer.setBuffer(Collections.synchronizedList(new LinkedList<>()));
- } else {
- rowBuffer.setBuffer(Collections.synchronizedList(new ArrayList<>()));
- }
- targetMap.put(key, rowBuffer);
- }
+ /**
+ * Create the sampler for one transform or hop. Package-private for tests.
+ *
+ * @param sampleType First, Last or Random
+ * @param sampleSize the number of rows to keep
+ * @param rowMeta the layout of the sampled rows
+ * @return a new sampler with an empty buffer
+ */
+ static RowSampler createSampler(SampleType sampleType, int sampleSize,
IRowMeta rowMeta) {
+ return switch (sampleType) {
+ case First -> new FirstRowsSampler(rowMeta, sampleSize);
+ case Last -> new LastRowsSampler(rowMeta, sampleSize);
+ case Random -> new RandomRowsSampler(rowMeta, sampleSize);
+ default -> throw new IllegalArgumentException("Unsupported sample type "
+ sampleType);
+ };
+ }
+
+ private static Object[] copyRow(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
try {
- row = rowMeta.cloneRow(row);
+ return rowMeta.cloneRow(row);
} catch (HopValueException e) {
throw new HopTransformException("Error copying row for preview", e);
}
- switch (sampleType) {
- case First:
- if (rowBuffer.size() < sampleSize) {
- rowBuffer.addRow(row);
+ }
+
+ /**
+ * Keeps a sample of the rows of one transform or hop in a {@link
RowBuffer}. Readers lock the
+ * buffer's list ({@code synchronized (rowBuffer.getBuffer())}) to get a
consistent view, so every
+ * change to the list happens under that same lock.
+ */
+ abstract static class RowSampler {
+ protected final RowBuffer rowBuffer;
+ protected final int sampleSize;
+
+ RowSampler(RowBuffer rowBuffer, int sampleSize) {
+ this.rowBuffer = rowBuffer;
+ this.sampleSize = sampleSize;
+ }
+
+ RowBuffer getRowBuffer() {
+ return rowBuffer;
+ }
+
+ abstract void sample(IRowMeta rowMeta, Object[] row) throws
HopTransformException;
+ }
+
+ /** Keeps the first rows, then ignores the rest without cloning them. */
+ static final class FirstRowsSampler extends RowSampler {
+ private volatile boolean full;
+
+ FirstRowsSampler(IRowMeta rowMeta, int sampleSize) {
+ super(new RowBuffer(rowMeta), sampleSize);
+ }
+
+ @Override
+ void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ if (full) {
+ return;
+ }
+ List<Object[]> rows = rowBuffer.getBuffer();
+ synchronized (rows) {
+ if (rows.size() < sampleSize) {
+ rows.add(copyRow(rowMeta, row));
}
- break;
- case Last:
- rowBuffer.addRow(0, row);
- if (rowBuffer.size() > sampleSize) {
- rowBuffer.removeRow(rowBuffer.size() - 1);
+ full = rows.size() >= sampleSize;
+ }
+ }
+ }
+
+ /** Keeps the last rows in a fixed ring, newest first. */
+ static final class LastRowsSampler extends RowSampler {
+ private final LastRowsList rows;
+
+ LastRowsSampler(IRowMeta rowMeta, int sampleSize) {
+ super(new RowBuffer(rowMeta), sampleSize);
+ rows = new LastRowsList(sampleSize);
+ rowBuffer.setBuffer(rows);
+ }
+
+ @Override
+ void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ rows.addNewest(copyRow(rowMeta, row));
+ }
+ }
+
+ /**
+ * Keeps a uniform random sample (reservoir sampling). Only the rows that
enter the reservoir are
+ * cloned.
+ */
+ static final class RandomRowsSampler extends RowSampler {
+ private long rowsSeen;
+
+ RandomRowsSampler(IRowMeta rowMeta, int sampleSize) {
+ super(new RowBuffer(rowMeta), sampleSize);
+ }
+
+ @Override
+ void sample(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ List<Object[]> rows = rowBuffer.getBuffer();
+ synchronized (rows) {
+ long seen = ++rowsSeen;
+ if (seen <= sampleSize) {
+ rows.add(copyRow(rowMeta, row));
+ return;
}
- break;
- case Random:
- int nrRows = rowCounters.computeIfAbsent(key, k -> new
AtomicInteger(0)).incrementAndGet();
- if (rowBuffer.size() < sampleSize) {
- rowBuffer.addRow(row);
- } else {
- int randomIndex = random.nextInt(nrRows);
- if (randomIndex < sampleSize) {
- rowBuffer.setRow(randomIndex, row);
- }
+ long index = ThreadLocalRandom.current().nextLong(seen);
+ if (index < sampleSize) {
+ rows.set((int) index, copyRow(rowMeta, row));
}
- break;
- default:
- break;
+ }
+ }
+ }
+
+ /**
+ * A fixed-size ring of rows exposed as a list, newest row at index 0.
Adding a row once the ring
+ * is full replaces the oldest one, without allocating a node or shifting
elements. All access
+ * locks the list itself, the same lock {@code Collections.synchronizedList}
uses, so readers can
+ * keep doing {@code synchronized (rowBuffer.getBuffer())}.
+ */
+ static final class LastRowsList extends AbstractList<Object[]> {
+ private final Object[][] ring;
+ private int next;
+ private int size;
+
+ LastRowsList(int capacity) {
+ ring = new Object[capacity][];
+ }
+
+ synchronized void addNewest(Object[] row) {
+ ring[next] = row;
+ next = (next + 1) % ring.length;
+ if (size < ring.length) {
+ size++;
+ }
+ }
+
+ @Override
+ public synchronized Object[] get(int index) {
+ if (index < 0 || index >= size) {
+ throw new IndexOutOfBoundsException("Index " + index + ", size " +
size);
+ }
+ return ring[Math.floorMod(next - 1 - index, ring.length)];
+ }
+
+ @Override
+ public synchronized int size() {
+ return size;
+ }
+
+ @Override
+ public synchronized void clear() {
+ Arrays.fill(ring, null);
+ next = 0;
+ size = 0;
+ modCount++;
}
}
}
diff --git
a/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
new file mode 100644
index 0000000000..47da41ca0e
--- /dev/null
+++
b/ui/src/test/java/org/apache/hop/ui/hopgui/file/shared/PipelineRowSamplerHelperTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.file.shared;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import
org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration.SampleType;
+import
org.apache.hop.ui.hopgui.file.shared.PipelineRowSamplerHelper.RowSampler;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class PipelineRowSamplerHelperTest {
+
+ private static final int SAMPLE_SIZE = 10;
+
+ private IRowMeta rowMeta;
+
+ @BeforeEach
+ void setUp() {
+ rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("nr"));
+ rowMeta.addValueMeta(new ValueMetaDate("date"));
+ }
+
+ private static Object[] row(long nr) {
+ return new Object[] {nr, new Date(nr)};
+ }
+
+ private static List<Long> numbers(RowSampler sampler) {
+ List<Object[]> rows = sampler.getRowBuffer().getBuffer();
+ List<Long> numbers = new ArrayList<>();
+ synchronized (rows) {
+ for (Object[] r : rows) {
+ numbers.add((Long) r[0]);
+ }
+ }
+ return numbers;
+ }
+
+ private RowSampler feed(SampleType sampleType, int rowCount) throws
Exception {
+ RowSampler sampler = PipelineRowSamplerHelper.createSampler(sampleType,
SAMPLE_SIZE, rowMeta);
+ for (long nr = 1; nr <= rowCount; nr++) {
+ sampler.sample(rowMeta, row(nr));
+ }
+ return sampler;
+ }
+
+ @Test
+ void firstKeepsTheFirstRows() throws Exception {
+ assertEquals(List.of(1L, 2L, 3L), numbers(feed(SampleType.First, 3)));
+ assertEquals(
+ List.of(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L),
numbers(feed(SampleType.First, 1000)));
+ }
+
+ @Test
+ void lastKeepsTheLastRowsNewestFirst() throws Exception {
+ assertEquals(List.of(3L, 2L, 1L), numbers(feed(SampleType.Last, 3)));
+ assertEquals(
+ List.of(1000L, 999L, 998L, 997L, 996L, 995L, 994L, 993L, 992L, 991L),
+ numbers(feed(SampleType.Last, 1000)));
+ }
+
+ @Test
+ void lastAtExactlyTheSampleSize() throws Exception {
+ assertEquals(
+ List.of(10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L, 1L),
+ numbers(feed(SampleType.Last, SAMPLE_SIZE)));
+ assertEquals(
+ List.of(11L, 10L, 9L, 8L, 7L, 6L, 5L, 4L, 3L, 2L),
+ numbers(feed(SampleType.Last, SAMPLE_SIZE + 1)));
+ }
+
+ @Test
+ void randomKeepsAllRowsWhileThereAreFewerThanTheSampleSize() throws
Exception {
+ assertEquals(List.of(1L, 2L, 3L), numbers(feed(SampleType.Random, 3)));
+ }
+
+ @Test
+ void randomKeepsSampleSizeDistinctInputRows() throws Exception {
+ List<Long> numbers = numbers(feed(SampleType.Random, 10_000));
+ assertEquals(SAMPLE_SIZE, numbers.size());
+ Set<Long> distinct = new HashSet<>(numbers);
+ assertEquals(SAMPLE_SIZE, distinct.size());
+ for (long nr : numbers) {
+ assertTrue(nr >= 1 && nr <= 10_000, "row " + nr + " was never written");
+ }
+ // With 10,000 rows, a sample made only of the first rows means the
reservoir never replaced
+ assertTrue(numbers.stream().anyMatch(nr -> nr > SAMPLE_SIZE));
+ }
+
+ @Test
+ void keptRowsAreCopies() throws Exception {
+ for (SampleType sampleType : List.of(SampleType.First, SampleType.Last,
SampleType.Random)) {
+ RowSampler sampler = PipelineRowSamplerHelper.createSampler(sampleType,
SAMPLE_SIZE, rowMeta);
+ Object[] written = row(42);
+ sampler.sample(rowMeta, written);
+
+ // A later transform may replace values in the array it received, as
Value Mapper does
+ written[0] = 0L;
+ written[1] = new Date(0L);
+
+ Object[] kept = sampler.getRowBuffer().getBuffer().get(0);
+ assertNotSame(written, kept, sampleType.name());
+ assertEquals(42L, kept[0], sampleType.name());
+ assertEquals(42L, ((Date) kept[1]).getTime(), sampleType.name());
+ }
+ }
+}