This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new b66b54e40f Memory group by fails with a NullPointerException on a null
value in a standard deviation, fixes #9299 (#9300)
b66b54e40f is described below
commit b66b54e40fc230fcc74e9a30f75fcf532dc5ea6c
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri Nov 6 07:05:38 2026 +0100
Memory group by fails with a NullPointerException on a null value in a
standard deviation, fixes #9299 (#9300)
---
.../transforms/memgroupby/MemoryGroupBy.java | 35 ++++---
.../MemoryGroupByStandardDeviationTest.java | 106 +++++++++++++++++++++
2 files changed, 126 insertions(+), 15 deletions(-)
diff --git
a/plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupBy.java
b/plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupBy.java
index a11ecca4d6..abb5b8e788 100644
---
a/plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupBy.java
+++
b/plugins/transforms/memgroupby/src/main/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupBy.java
@@ -280,19 +280,21 @@ public class MemoryGroupBy extends
BaseTransform<MemoryGroupByMeta, MemoryGroupB
if (aggregate.mean == null) {
aggregate.mean = new double[meta.getAggregates().size()];
}
- aggregate.counts[i]++;
- double n = aggregate.counts[i];
- double x = subjMeta.getNumber(subj);
- // for standard deviation null is exact 0
- double sum = value == null ? Double.valueOf(0) : (Double) value;
- double mean = aggregate.mean[i];
-
- double delta = x - mean;
- mean = mean + (delta / n);
- sum = sum + delta * (x - mean);
-
- aggregate.mean[i] = mean;
- aggregate.agg[i] = sum;
+ if (!subjMeta.isNull(subj)) {
+ aggregate.counts[i]++;
+ double n = aggregate.counts[i];
+ double x = subjMeta.getNumber(subj);
+ // for standard deviation null is exact 0
+ double sum = value == null ? Double.valueOf(0) : (Double) value;
+ double mean = aggregate.mean[i];
+
+ double delta = x - mean;
+ mean = mean + (delta / n);
+ sum = sum + delta * (x - mean);
+
+ aggregate.mean[i] = mean;
+ aggregate.agg[i] = sum;
+ }
break;
case CountDistinct:
if (aggregate.distinctObjs == null) {
@@ -518,8 +520,11 @@ public class MemoryGroupBy extends
BaseTransform<MemoryGroupByMeta, MemoryGroupB
case Maximum:
break;
case StandardDeviation:
- double sum = (Double) ag / aggregate.counts[i];
- ag = Math.sqrt(sum);
+ if (ag != null) {
+ // all values null: no deviation, like Group by
+ double sum = (Double) ag / aggregate.counts[i];
+ ag = Math.sqrt(sum);
+ }
break;
case ConcatComma, ConcatString:
ag = ((StringBuilder) ag).toString();
diff --git
a/plugins/transforms/memgroupby/src/test/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupByStandardDeviationTest.java
b/plugins/transforms/memgroupby/src/test/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupByStandardDeviationTest.java
new file mode 100644
index 0000000000..455e9a642c
--- /dev/null
+++
b/plugins/transforms/memgroupby/src/test/java/org/apache/hop/pipeline/transforms/memgroupby/MemoryGroupByStandardDeviationTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.memgroupby;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import
org.apache.hop.pipeline.transforms.memgroupby.MemoryGroupByMeta.GroupType;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Standard deviation skips null values, like the Group by transform does. */
+class MemoryGroupByStandardDeviationTest {
+
+ static TransformMockHelper<MemoryGroupByMeta, MemoryGroupByData> mockHelper;
+
+ private MemoryGroupBy transform;
+ private MemoryGroupByData data;
+
+ @BeforeAll
+ static void setUpBeforeClass() {
+ mockHelper =
+ new TransformMockHelper<>(
+ "Memory Group By", MemoryGroupByMeta.class,
MemoryGroupByData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ when(mockHelper.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterAll
+ static void cleanUp() {
+ mockHelper.cleanUp();
+ }
+
+ @BeforeEach
+ void setUp() {
+ MemoryGroupByMeta meta = new MemoryGroupByMeta();
+ meta.setDefault();
+ meta.getAggregates().add(new GAggregate("sd", "x",
GroupType.StandardDeviation, null));
+ when(mockHelper.transformMeta.getTransform()).thenReturn(meta);
+
+ data = new MemoryGroupByData();
+ data.inputRowMeta = new RowMeta();
+ data.inputRowMeta.addValueMeta(new ValueMetaNumber("x"));
+ data.groupMeta = new RowMeta();
+ data.groupnrs = new int[] {};
+ data.subjectnrs = new int[] {0};
+ data.aggMeta = new RowMeta();
+ data.aggMeta.addValueMeta(new ValueMetaNumber("sd"));
+ data.map = new HashMap<>();
+
+ transform =
+ new MemoryGroupBy(
+ mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta,
mockHelper.pipeline);
+ }
+
+ private Aggregate theAggregate() {
+ return data.map.get(data.getHashEntry(new Object[0]));
+ }
+
+ @Test
+ void nullValuesAreSkipped() throws Exception {
+ transform.addToAggregate(new Object[] {2.0});
+ transform.addToAggregate(new Object[] {null});
+ transform.addToAggregate(new Object[] {4.0});
+
+ Object[] result = transform.getAggregateResult(theAggregate());
+
+ // population standard deviation of {2, 4}
+ assertEquals(1.0, (Double) result[0], 1e-9);
+ }
+
+ @Test
+ void onlyNullValuesGiveNull() throws Exception {
+ transform.addToAggregate(new Object[] {null});
+ transform.addToAggregate(new Object[] {null});
+
+ Object[] result = transform.getAggregateResult(theAggregate());
+
+ assertNull(result[0]);
+ }
+}