This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new f52271e35d9 SOLR-18328 Add support for standard deviation in rollup
for streaming expressions (#4691)
f52271e35d9 is described below
commit f52271e35d9753a976de2108331d839d77dde6b7
Author: Khush Jain <[email protected]>
AuthorDate: Mon Aug 10 10:39:15 2026 -0400
SOLR-18328 Add support for standard deviation in rollup for streaming
expressions (#4691)
(cherry picked from commit 5c28547dd82775b387b4a6ffa75fb5672318ce85)
---
.../SOLR-18328-support-std-in-rollup.yml | 7 +++
.../pages/stream-decorator-reference.adoc | 5 +-
.../client/solrj/io/stream/metrics/StdMetric.java | 43 +++++++++++++----
.../solrj/io/stream/StreamExpressionTest.java | 56 ++++++++++++++++++++++
.../solr/client/solrj/io/stream/StreamingTest.java | 15 ++++++
5 files changed, 114 insertions(+), 12 deletions(-)
diff --git a/changelog/unreleased/SOLR-18328-support-std-in-rollup.yml
b/changelog/unreleased/SOLR-18328-support-std-in-rollup.yml
new file mode 100644
index 00000000000..dbaf0175293
--- /dev/null
+++ b/changelog/unreleased/SOLR-18328-support-std-in-rollup.yml
@@ -0,0 +1,7 @@
+title: "Support 'std' (standard deviation) metric in rollup for streaming
expressions"
+type: added
+authors:
+ - name: khushjain
+links:
+ - name: SOLR-18328
+ url: https://issues.apache.org/jira/browse/SOLR-18328
diff --git
a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc
b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc
index 9cbc8e440be..d9cadb16c49 100644
---
a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc
+++
b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc
@@ -1448,7 +1448,7 @@ For faster aggregation over low to moderate cardinality
fields, the `facet` func
* `StreamExpression` (Mandatory)
* `over`: (Mandatory) A list of fields to group by.
* `metrics`: (Mandatory) The list of metrics to compute.
-Currently supported metrics are `sum(col)`, `avg(col)`, `min(col)`,
`max(col)`, `count(*)`, `missing(col)`, `countDist(col)`, `per(col,
percentile)`.
+Currently supported metrics are `sum(col)`, `avg(col)`, `min(col)`,
`max(col)`, `count(*)`, `missing(col)`, `countDist(col)`, `per(col,
percentile)`, `std(col)`.
=== rollup Syntax
@@ -1469,7 +1469,8 @@ rollup(
missing(a_i),
countDist(a_i),
per(a_i, 50),
- per(a_f, 75)
+ per(a_f, 75),
+ std(a_i)
)
----
diff --git
a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/metrics/StdMetric.java
b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/metrics/StdMetric.java
index 57ca2b08cac..697e638406f 100644
---
a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/metrics/StdMetric.java
+++
b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/metrics/StdMetric.java
@@ -23,16 +23,15 @@ import
org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+/**
+ * Metric that computes the sample standard deviation of a numeric column over
a stream. Consistent
+ * with the {@code std} streaming evaluator.
+ */
public class StdMetric extends Metric {
- // How'd the MeanMetric get to be so mean?
- // Maybe it was born with it.
- // Maybe it was mayba-mean.
- //
- // I'll see myself out.
private String columnName;
- private double doubleSum;
- private long longSum;
+ private double sum;
+ private double sumSq;
private long count;
public StdMetric(String columnName) {
@@ -75,11 +74,28 @@ public class StdMetric extends Metric {
}
@Override
- public void update(Tuple tuple) {}
+ public void update(Tuple tuple) {
+ Object o = tuple.get(columnName);
+ double val;
+ if (o instanceof Double d) {
+ val = d;
+ } else if (o instanceof Float f) {
+ val = f.doubleValue();
+ } else if (o instanceof Integer i) {
+ val = i.doubleValue();
+ } else if (o instanceof Long l) {
+ val = l.doubleValue();
+ } else {
+ return;
+ }
+ ++count;
+ sum += val;
+ sumSq += val * val;
+ }
@Override
public Metric newInstance() {
- return new MeanMetric(columnName, outputLong);
+ return new StdMetric(columnName, outputLong);
}
@Override
@@ -87,9 +103,16 @@ public class StdMetric extends Metric {
return new String[] {columnName};
}
+ /** Returns the sample standard deviation of the values seen so far. */
@Override
public Number getValue() {
- return null;
+ double std =
+ count <= 1 ? 0.0d : Math.sqrt(((count * sumSq) - (sum * sum)) / (count
* (count - 1.0D)));
+ if (outputLong) {
+ return Math.round(std);
+ } else {
+ return std;
+ }
}
@Override
diff --git
a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 4ef5a73ccef..de80b4ae728 100644
---
a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++
b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -1333,6 +1333,62 @@ public class StreamExpressionTest extends
SolrCloudTestCase {
assertEquals(saf, 18, 0);
}
+ @Test
+ public void testRollupStdMetric() throws Exception {
+ new UpdateRequest()
+ .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1")
+ .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2")
+ .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3")
+ .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4")
+ .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5")
+ .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6")
+ .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7")
+ .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8")
+ .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9")
+ .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ String expr =
+ "rollup("
+ + " search(collection1, q=*:*, fl=\"a_s,a_i,a_f\", sort=\"a_s
asc\", qt=\"/export\"),"
+ + " over=\"a_s\", std(a_i), std(a_f), count(*)"
+ + ")";
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url =
+ cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" +
COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+
+ assertEquals(3, tuples.size());
+
+ // hello0: a_i = [0, 1, 2, 14], a_f = [1, 2, 5, 10]
+ Tuple tuple = tuples.get(0);
+ assertEquals("hello0", tuple.getString("a_s"));
+ assertEquals(6.5511, tuple.getDouble("std(a_i)"), 0.001);
+ assertEquals(4.0415, tuple.getDouble("std(a_f)"), 0.001);
+ assertEquals(4, tuple.getDouble("count(*)"), 0.0);
+
+ // hello3: a_i = [3, 10, 12, 13], a_f = [3, 6, 8, 9]
+ tuple = tuples.get(1);
+ assertEquals("hello3", tuple.getString("a_s"));
+ assertEquals(4.5092, tuple.getDouble("std(a_i)"), 0.001);
+ assertEquals(2.6458, tuple.getDouble("std(a_f)"), 0.001);
+ assertEquals(4, tuple.getDouble("count(*)"), 0.0);
+
+ // hello4: a_i = [4, 11], a_f = [4, 7]
+ tuple = tuples.get(2);
+ assertEquals("hello4", tuple.getString("a_s"));
+ assertEquals(4.9497, tuple.getDouble("std(a_i)"), 0.001);
+ assertEquals(2.1213, tuple.getDouble("std(a_f)"), 0.001);
+ assertEquals(2, tuple.getDouble("count(*)"), 0.0);
+ }
+
@Test
public void testFacetStream() throws Exception {
diff --git
a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
index 2ce5c3d7afd..31bf75c7b18 100644
---
a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
+++
b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
@@ -51,6 +51,7 @@ import org.apache.solr.client.solrj.io.stream.metrics.Metric;
import org.apache.solr.client.solrj.io.stream.metrics.MinMetric;
import org.apache.solr.client.solrj.io.stream.metrics.MissingMetric;
import org.apache.solr.client.solrj.io.stream.metrics.PercentileMetric;
+import org.apache.solr.client.solrj.io.stream.metrics.StdMetric;
import org.apache.solr.client.solrj.io.stream.metrics.SumMetric;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -1676,6 +1677,8 @@ public class StreamingTest extends SolrCloudTestCase {
new MaxMetric("a_f"),
new MeanMetric("a_i"),
new MeanMetric("a_f"),
+ new StdMetric("a_i"),
+ new StdMetric("a_f"),
new CountMetric(),
new MissingMetric("b_f"),
new CountDistinctMetric("a_i"),
@@ -1700,6 +1703,8 @@ public class StreamingTest extends SolrCloudTestCase {
Double maxf = tuple.getDouble("max(a_f)");
Double avgi = tuple.getDouble("avg(a_i)");
Double avgf = tuple.getDouble("avg(a_f)");
+ Double stdi = tuple.getDouble("std(a_i)");
+ Double stdf = tuple.getDouble("std(a_f)");
Double count = tuple.getDouble("count(*)");
Double missingBf = tuple.getDouble("missing(b_f)");
Double countDistI = tuple.getDouble("countDist(a_i)");
@@ -1714,6 +1719,8 @@ public class StreamingTest extends SolrCloudTestCase {
assertEquals(10, maxf, 0.001);
assertEquals(4.25, avgi, 0.001);
assertEquals(4.5, avgf, 0.001);
+ assertEquals(6.5511, stdi, 0.001);
+ assertEquals(4.0415, stdf, 0.001);
assertEquals(4, count, 0.001);
assertEquals(2, missingBf, 0.001);
assertEquals(4, countDistI, 0.001);
@@ -1729,6 +1736,8 @@ public class StreamingTest extends SolrCloudTestCase {
maxf = tuple.getDouble("max(a_f)");
avgi = tuple.getDouble("avg(a_i)");
avgf = tuple.getDouble("avg(a_f)");
+ stdi = tuple.getDouble("std(a_i)");
+ stdf = tuple.getDouble("std(a_f)");
count = tuple.getDouble("count(*)");
missingBf = tuple.getDouble("missing(b_f)");
countDistI = tuple.getDouble("countDist(a_i)");
@@ -1743,6 +1752,8 @@ public class StreamingTest extends SolrCloudTestCase {
assertEquals(9, maxf, 0.001);
assertEquals(9.5, avgi, 0.001);
assertEquals(6.5, avgf, 0.001);
+ assertEquals(4.5092, stdi, 0.001);
+ assertEquals(2.6458, stdf, 0.001);
assertEquals(4, count, 0.001);
assertEquals(3, missingBf, 0.001);
assertEquals(4, countDistI, 0.001);
@@ -1758,6 +1769,8 @@ public class StreamingTest extends SolrCloudTestCase {
maxf = tuple.getDouble("max(a_f)");
avgi = tuple.getDouble("avg(a_i)");
avgf = tuple.getDouble("avg(a_f)");
+ stdi = tuple.getDouble("std(a_i)");
+ stdf = tuple.getDouble("std(a_f)");
count = tuple.getDouble("count(*)");
missingBf = tuple.getDouble("missing(b_f)");
countDistI = tuple.getDouble("countDist(a_i)");
@@ -1772,6 +1785,8 @@ public class StreamingTest extends SolrCloudTestCase {
assertEquals(7, maxf, 0.01);
assertEquals(7.5, avgi, 0.01);
assertEquals(5.5, avgf, 0.01);
+ assertEquals(4.9497, stdi, 0.01);
+ assertEquals(2.1213, stdf, 0.01);
assertEquals(2, count, 0.01);
assertEquals(0, missingBf, 0.01);
assertEquals(2, countDistI, 0.01);
