This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch feature/SOLR-17458-rebased
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/feature/SOLR-17458-rebased by
this push:
new 315bf57ee49 SOLR-17865: Support aggregateNodeLevelMetricsEnabled
(#3734)
315bf57ee49 is described below
commit 315bf57ee4937830f1314f51b2ddda1ae3fed092
Author: Matthew Biscocho <[email protected]>
AuthorDate: Mon Oct 13 23:09:09 2025 -0400
SOLR-17865: Support aggregateNodeLevelMetricsEnabled (#3734)
Aggregates some core metrics to the node level.
---------
Co-authored-by: Matthew Biscocho <[email protected]>
---
.../apache/solr/handler/RequestHandlerBase.java | 95 +++++------
.../solr/handler/component/SearchHandler.java | 5 +-
.../otel/instruments/AttributedDoubleCounter.java | 2 +-
.../instruments/AttributedDoubleUpDownCounter.java | 4 +-
.../instruments/AttributedInstrumentFactory.java | 160 ++++++++++++++++++
.../otel/instruments/AttributedLongCounter.java | 2 +-
.../instruments/AttributedLongUpDownCounter.java | 4 +-
...java => DualRegistryAttributedLongCounter.java} | 28 ++--
...r.java => DualRegistryAttributedLongTimer.java} | 32 ++--
...> DualRegistryAttributedLongUpDownCounter.java} | 32 ++--
.../apache/solr/update/DirectUpdateHandler2.java | 175 ++++++++++----------
.../conf/solrconfig.xml | 4 +-
.../solr/handler/RequestHandlerBaseTest.java | 3 +-
.../solr/handler/RequestHandlerMetricsTest.java | 183 +++++++++------------
14 files changed, 433 insertions(+), 296 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index f726842d773..f2407a4273e 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -21,8 +21,6 @@ import static
org.apache.solr.response.SolrQueryResponse.haveCompleteResults;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.metrics.LongCounter;
-import io.opentelemetry.api.metrics.LongHistogram;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
@@ -41,11 +39,10 @@ import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;
import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.metrics.otel.OtelUnit;
+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;
import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;
import org.apache.solr.metrics.otel.instruments.AttributedLongTimer;
import org.apache.solr.request.SolrQueryRequest;
@@ -72,6 +69,7 @@ public abstract class RequestHandlerBase
PermissionNameProvider {
public static final String REQUEST_CPU_TIMER_CONTEXT = "publishCpuTime";
+ public static final AttributeKey<String> SOURCE_ATTR =
AttributeKey.stringKey("source");
protected NamedList<?> initArgs = null;
protected SolrParams defaults;
protected SolrParams appends;
@@ -167,21 +165,13 @@ public abstract class RequestHandlerBase
@Override
public void initializeMetrics(
SolrMetricsContext parentContext, Attributes attributes, String scope) {
- if (aggregateNodeLevelMetricsEnabled) {
- this.solrMetricsContext =
- new SolrDelegateRegistryMetricsContext(
- parentContext.getMetricManager(),
- parentContext.getRegistryName(),
- SolrMetricProducer.getUniqueMetricTag(this,
parentContext.getTag()),
- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
- } else {
- this.solrMetricsContext = parentContext.getChildContext(this);
- }
+ this.solrMetricsContext = parentContext.getChildContext(this);
metrics =
new HandlerMetrics(
solrMetricsContext,
- attributes.toBuilder().put(CATEGORY_ATTR,
getCategory().toString()).build());
+ attributes.toBuilder().put(CATEGORY_ATTR,
getCategory().toString()).build(),
+ aggregateNodeLevelMetricsEnabled);
}
/** Metrics for this handler. */
@@ -189,7 +179,8 @@ public abstract class RequestHandlerBase
public static final HandlerMetrics NO_OP =
new HandlerMetrics(
new SolrMetricsContext(new SolrMetricManager(null), "NO_OP",
"NO_OP"),
- Attributes.empty());
+ Attributes.empty(),
+ false);
public AttributedLongCounter requests;
public AttributedLongCounter numServerErrors;
@@ -197,53 +188,41 @@ public abstract class RequestHandlerBase
public AttributedLongCounter numTimeouts;
public AttributedLongTimer requestTimes;
- public HandlerMetrics(SolrMetricsContext solrMetricsContext, Attributes
attributes) {
- LongCounter requestMetric;
- LongCounter errorRequestMetric;
- LongCounter timeoutRequestMetric;
- LongHistogram requestTimeMetric;
-
- if (solrMetricsContext.getRegistryName().equals("solr.node")) {
- requestMetric =
- solrMetricsContext.longCounter("solr_node_requests", "Http Solr
node requests");
- errorRequestMetric =
- solrMetricsContext.longCounter(
- "solr_node_requests_errors", "HTTP Solr node request errors");
- timeoutRequestMetric =
- solrMetricsContext.longCounter(
- "solr_node_requests_timeout", "HTTP Solr node request
timeouts");
- requestTimeMetric =
- solrMetricsContext.longHistogram(
- "solr_node_requests_times", "HTTP Solr node request times",
OtelUnit.MILLISECONDS);
- } else {
- requestMetric =
- solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr
core requests");
- errorRequestMetric =
- solrMetricsContext.longCounter(
- "solr_core_requests_errors", "HTTP Solr core request errors");
- timeoutRequestMetric =
- solrMetricsContext.longCounter(
- "solr_core_requests_timeout", "HTTP Solr core request
timeouts");
- requestTimeMetric =
- solrMetricsContext.longHistogram(
- "solr_core_requests_times", "HTTP Solr core request times",
OtelUnit.MILLISECONDS);
- }
+ public HandlerMetrics(
+ SolrMetricsContext solrMetricsContext,
+ Attributes coreAttributes,
+ boolean aggregateNodeLevelMetricsEnabled) {
- requests = new AttributedLongCounter(requestMetric, attributes);
+ AttributedInstrumentFactory factory =
+ new AttributedInstrumentFactory(
+ solrMetricsContext, coreAttributes,
aggregateNodeLevelMetricsEnabled);
+
+ requests =
+ factory.attributedLongCounter(
+ "solr_core_requests", "HTTP Solr requests", Attributes.empty());
numServerErrors =
- new AttributedLongCounter(
- errorRequestMetric,
- attributes.toBuilder().put(AttributeKey.stringKey("source"),
"server").build());
+ factory.attributedLongCounter(
+ "solr_core_requests_errors",
+ "HTTP Solr request errors",
+ Attributes.of(SOURCE_ATTR, "server"));
numClientErrors =
- new AttributedLongCounter(
- errorRequestMetric,
- attributes.toBuilder().put(AttributeKey.stringKey("source"),
"client").build());
-
- numTimeouts = new AttributedLongCounter(timeoutRequestMetric,
attributes);
-
- requestTimes = new AttributedLongTimer(requestTimeMetric, attributes);
+ factory.attributedLongCounter(
+ "solr_core_requests_errors",
+ "HTTP Solr request errors",
+ Attributes.of(SOURCE_ATTR, "client"));
+
+ numTimeouts =
+ factory.attributedLongCounter(
+ "solr_core_requests_timeout", "HTTP Solr request timeouts",
Attributes.empty());
+
+ requestTimes =
+ factory.attributedLongTimer(
+ "solr_core_requests_times",
+ "HTTP Solr request times",
+ OtelUnit.MILLISECONDS,
+ Attributes.empty());
}
}
diff --git
a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index b42e62d187b..cc7ad157651 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -98,8 +98,6 @@ public class SearchHandler extends RequestHandlerBase
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
- protected static final String SHARD_HANDLER_SUFFIX = "[shard]";
-
private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
@@ -171,7 +169,8 @@ public class SearchHandler extends RequestHandlerBase
.putAll(attributes)
.put(CATEGORY_ATTR, getCategory().toString())
.put(INTERNAL_ATTR, true)
- .build());
+ .build(),
+ false);
}
@Override
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
index 43adf8ad087..e2f28293e4a 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java
@@ -29,7 +29,7 @@ public class AttributedDoubleCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1.0);
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
index 23c0d3b9a53..ca2808a596f 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java
@@ -29,11 +29,11 @@ public class AttributedDoubleUpDownCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1.0);
}
- public void dec() {
+ public final void dec() {
add(-1.0);
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java
new file mode 100644
index 00000000000..a443eab92b0
--- /dev/null
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import static org.apache.solr.handler.component.SearchHandler.INTERNAL_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR;
+import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongHistogram;
+import io.opentelemetry.api.metrics.LongUpDownCounter;
+import java.util.Set;
+import org.apache.solr.core.SolrInfoBean;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.metrics.otel.OtelUnit;
+
+/**
+ * Factory for creating metrics instruments that can write to either single or
dual registries (core
+ * and node).
+ */
+public class AttributedInstrumentFactory {
+
+ private static final Set<AttributeKey<?>> FILTER_ATTRS_SET =
+ Set.of(COLLECTION_ATTR, CORE_ATTR, SHARD_ATTR, REPLICA_TYPE_ATTR,
INTERNAL_ATTR);
+ private final SolrMetricsContext primaryMetricsContext;
+ private final Attributes primaryAttributes;
+ private final boolean aggregateToNodeRegistry;
+ private final boolean primaryIsNodeRegistry;
+ private SolrMetricsContext nodeMetricsContext = null;
+ private Attributes nodeAttributes = null;
+
+ public AttributedInstrumentFactory(
+ SolrMetricsContext primaryMetricsContext,
+ Attributes primaryAttributes,
+ boolean aggregateToNodeRegistry) {
+ this.primaryMetricsContext = primaryMetricsContext;
+ this.primaryAttributes = primaryAttributes;
+ this.aggregateToNodeRegistry = aggregateToNodeRegistry;
+
+ // Primary could be a node
+ this.primaryIsNodeRegistry =
+ primaryMetricsContext
+ .getRegistryName()
+
.equals(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
+
+ // Only create node registry if we want dual registry mode AND primary is
not already a node
+ // registry
+ if (aggregateToNodeRegistry && !primaryIsNodeRegistry) {
+ this.nodeMetricsContext =
+ new SolrMetricsContext(
+ primaryMetricsContext.getMetricManager(),
+ SolrMetricManager.getRegistryName(SolrInfoBean.Group.node),
+ null);
+ this.nodeAttributes = createNodeAttributes(primaryAttributes);
+ }
+ }
+
+ public AttributedLongCounter attributedLongCounter(
+ String metricName, String description, Attributes additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes,
additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes,
additionalAttributes);
+
+ LongCounter primaryCounter =
primaryMetricsContext.longCounter(metricName, description);
+ LongCounter nodeCounter =
+ nodeMetricsContext.longCounter(toNodeMetricName(metricName),
description);
+ return new DualRegistryAttributedLongCounter(
+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ?
toNodeMetricName(metricName) : metricName;
+ LongCounter counter = primaryMetricsContext.longCounter(finalMetricName,
description);
+ return new AttributedLongCounter(counter, finalPrimaryAttrs);
+ }
+ }
+
+ public AttributedLongUpDownCounter attributedLongUpDownCounter(
+ String metricName, String description, Attributes additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes,
additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes,
additionalAttributes);
+
+ LongUpDownCounter primaryCounter =
+ primaryMetricsContext.longUpDownCounter(metricName, description);
+ LongUpDownCounter nodeCounter =
+ nodeMetricsContext.longUpDownCounter(toNodeMetricName(metricName),
description);
+ return new DualRegistryAttributedLongUpDownCounter(
+ primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ?
toNodeMetricName(metricName) : metricName;
+ LongUpDownCounter counter =
+ primaryMetricsContext.longUpDownCounter(finalMetricName,
description);
+ return new AttributedLongUpDownCounter(counter, finalPrimaryAttrs);
+ }
+ }
+
+ public AttributedLongTimer attributedLongTimer(
+ String metricName, String description, OtelUnit unit, Attributes
additionalAttributes) {
+ Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes,
additionalAttributes);
+
+ if (aggregateToNodeRegistry && nodeMetricsContext != null) {
+ Attributes finalNodeAttrs = appendAttributes(nodeAttributes,
additionalAttributes);
+ LongHistogram primaryHistogram =
+ primaryMetricsContext.longHistogram(metricName, description, unit);
+ LongHistogram nodeHistogram =
+ nodeMetricsContext.longHistogram(toNodeMetricName(metricName),
description, unit);
+ return new DualRegistryAttributedLongTimer(
+ primaryHistogram, finalPrimaryAttrs, nodeHistogram, finalNodeAttrs);
+ } else {
+ String finalMetricName = primaryIsNodeRegistry ?
toNodeMetricName(metricName) : metricName;
+ LongHistogram histogram =
+ primaryMetricsContext.longHistogram(finalMetricName, description,
unit);
+ return new AttributedLongTimer(histogram, finalPrimaryAttrs);
+ }
+ }
+
+ /** Replace core metric name prefix to node prefix */
+ private String toNodeMetricName(String coreMetricName) {
+ return coreMetricName.replace("solr_core", "solr_node");
+ }
+
+ /** Filter out core attributes and keep all others for node-level metrics */
+ @SuppressWarnings("unchecked")
+ private Attributes createNodeAttributes(Attributes coreAttributes) {
+ var builder = Attributes.builder();
+
+ coreAttributes.forEach(
+ (key, value) -> {
+ if (!FILTER_ATTRS_SET.contains(key)) {
+ builder.put((AttributeKey<Object>) key, value);
+ }
+ });
+
+ return builder.build();
+ }
+
+ private Attributes appendAttributes(Attributes base, Attributes additional) {
+ return base.toBuilder().putAll(additional).build();
+ }
+}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
index f33b4784c53..e2a477eac35 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
@@ -29,7 +29,7 @@ public class AttributedLongCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1L);
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
index 91487c9e677..e3ae3e98edd 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
@@ -29,11 +29,11 @@ public class AttributedLongUpDownCounter {
this.attributes = attributes;
}
- public void inc() {
+ public final void inc() {
add(1L);
}
- public void dec() {
+ public final void dec() {
add(-1L);
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java
similarity index 60%
copy from
solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
copy to
solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java
index f33b4784c53..15fc27b85c5 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java
@@ -19,21 +19,27 @@ package org.apache.solr.metrics.otel.instruments;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.metrics.LongCounter;
-public class AttributedLongCounter {
-
- private final LongCounter baseCounter;
- private final io.opentelemetry.api.common.Attributes attributes;
+/**
+ * An AttributedLongCounter that writes to both core and node registries with
corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongCounter extends AttributedLongCounter {
- public AttributedLongCounter(LongCounter baseCounter, Attributes attributes)
{
- this.baseCounter = baseCounter;
- this.attributes = attributes;
- }
+ private final AttributedLongCounter nodeCounter;
- public void inc() {
- add(1L);
+ public DualRegistryAttributedLongCounter(
+ LongCounter coreCounter,
+ Attributes coreAttributes,
+ LongCounter nodeCounter,
+ Attributes nodeAttributes) {
+ super(coreCounter, coreAttributes);
+ assert coreCounter != nodeCounter;
+ this.nodeCounter = new AttributedLongCounter(nodeCounter, nodeAttributes);
}
+ @Override
public void add(Long value) {
- baseCounter.add(value, attributes);
+ super.add(value);
+ nodeCounter.add(value);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java
similarity index 55%
copy from
solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
copy to
solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java
index f33b4784c53..b42d869892c 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java
@@ -17,23 +17,29 @@
package org.apache.solr.metrics.otel.instruments;
import io.opentelemetry.api.common.Attributes;
-import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongHistogram;
-public class AttributedLongCounter {
-
- private final LongCounter baseCounter;
- private final io.opentelemetry.api.common.Attributes attributes;
+/**
+ * An AttributedLongTimer that records to both core and node registries with
corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongTimer extends AttributedLongTimer {
- public AttributedLongCounter(LongCounter baseCounter, Attributes attributes)
{
- this.baseCounter = baseCounter;
- this.attributes = attributes;
- }
+ private final AttributedLongTimer nodeTimer;
- public void inc() {
- add(1L);
+ public DualRegistryAttributedLongTimer(
+ LongHistogram coreHistogram,
+ Attributes coreAttributes,
+ LongHistogram nodeHistogram,
+ Attributes nodeAttributes) {
+ super(coreHistogram, coreAttributes);
+ assert coreHistogram != nodeHistogram;
+ this.nodeTimer = new AttributedLongTimer(nodeHistogram, nodeAttributes);
}
- public void add(Long value) {
- baseCounter.add(value, attributes);
+ @Override
+ public void record(Long value) {
+ super.record(value);
+ nodeTimer.record(value);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java
similarity index 57%
copy from
solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
copy to
solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java
index 91487c9e677..6cf505fb54e 100644
---
a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java
+++
b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java
@@ -19,25 +19,27 @@ package org.apache.solr.metrics.otel.instruments;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.metrics.LongUpDownCounter;
-public class AttributedLongUpDownCounter {
-
- private final LongUpDownCounter upDownCounter;
- private final Attributes attributes;
-
- public AttributedLongUpDownCounter(LongUpDownCounter upDownCounter,
Attributes attributes) {
- this.upDownCounter = upDownCounter;
- this.attributes = attributes;
- }
+/**
+ * An AttributedLongUpDownCounter that writes to both core and node registries
with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongUpDownCounter extends
AttributedLongUpDownCounter {
- public void inc() {
- add(1L);
- }
+ private final AttributedLongUpDownCounter nodeUpDownCounter;
- public void dec() {
- add(-1L);
+ public DualRegistryAttributedLongUpDownCounter(
+ LongUpDownCounter coreUpDownCounter,
+ Attributes coreAttributes,
+ LongUpDownCounter nodeUpDownCounter,
+ Attributes nodeAttributes) {
+ super(coreUpDownCounter, coreAttributes);
+ assert coreUpDownCounter != nodeUpDownCounter;
+ this.nodeUpDownCounter = new
AttributedLongUpDownCounter(nodeUpDownCounter, nodeAttributes);
}
+ @Override
public void add(Long value) {
- upDownCounter.add(value, attributes);
+ super.add(value);
+ nodeUpDownCounter.add(value);
}
}
diff --git
a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index 0a027282c46..dc802afc2b7 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -53,11 +53,9 @@ import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;
-import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory;
import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;
import org.apache.solr.metrics.otel.instruments.AttributedLongUpDownCounter;
import org.apache.solr.request.LocalSolrQueryRequest;
@@ -227,84 +225,121 @@ public class DirectUpdateHandler2 extends UpdateHandler
@Override
public void initializeMetrics(
SolrMetricsContext parentContext, Attributes attributes, String scope) {
- if
(core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled) {
- this.solrMetricsContext =
- new SolrDelegateRegistryMetricsContext(
- parentContext.getMetricManager(),
- parentContext.getRegistryName(),
- SolrMetricProducer.getUniqueMetricTag(this,
parentContext.getTag()),
- SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
- } else {
- this.solrMetricsContext = parentContext.getChildContext(this);
- }
- final List<AutoCloseable> observables = new ArrayList<>();
+ this.solrMetricsContext = parentContext.getChildContext(this);
var baseAttributes =
attributes.toBuilder()
.put(AttributeKey.stringKey("category"), getCategory().toString())
.build();
- var baseCommandsMetric =
- solrMetricsContext.longUpDownCounter(
- "solr_core_update_cumulative_ops",
- "Cumulative number of update commands processed. Cumulative can
decrease from rollback command");
+ boolean aggregateNodeLevelMetricsEnabled =
+
core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled;
+
+ createMetrics(baseAttributes, aggregateNodeLevelMetricsEnabled);
+ }
+
+ private void createMetrics(Attributes baseAttributes, boolean
aggregateToNodeRegistry) {
+ final List<AutoCloseable> observables = new ArrayList<>();
+
+ AttributedInstrumentFactory factory =
+ new AttributedInstrumentFactory(
+ solrMetricsContext, baseAttributes, aggregateToNodeRegistry);
addCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR,
"adds").build());
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can
decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "adds"));
deleteByIdCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_id").build());
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can
decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
deleteByQueryCommandsCumulative =
- new AttributedLongUpDownCounter(
- baseCommandsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_query").build());
-
- var baseCommitOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_commit_ops", "Total number of commit
operations");
+ factory.attributedLongUpDownCounter(
+ "solr_core_update_cumulative_ops",
+ "Cumulative number of update commands processed. Cumulative can
decrease from rollback command",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
commitCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
baseAttributes.toBuilder().put(OPERATION_ATTR, "commits").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "commits"));
optimizeCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"optimize").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "optimize"));
mergeIndexesCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"merge_indexes").build());
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "merge_indexes"));
expungeDeleteCommands =
- new AttributedLongCounter(
- baseCommitOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"expunge_deletes").build());
-
- var baseMaintenanceMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_maintenance_ops", "Total number of maintenance
operations");
+ factory.attributedLongCounter(
+ "solr_core_update_commit_ops",
+ "Total number of commit operations",
+ Attributes.of(OPERATION_ATTR, "expunge_deletes"));
rollbackCommands =
- new AttributedLongCounter(
- baseMaintenanceMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"rollback").build());
+ factory.attributedLongCounter(
+ "solr_core_update_maintenance_ops",
+ "Total number of maintenance operations",
+ Attributes.of(OPERATION_ATTR, "rollback"));
splitCommands =
- new AttributedLongCounter(
- baseMaintenanceMetric,
baseAttributes.toBuilder().put(OPERATION_ATTR, "split").build());
-
- var baseErrorsMetric =
- solrMetricsContext.longCounter("solr_core_update_errors", "Total
number of update errors");
+ factory.attributedLongCounter(
+ "solr_core_update_maintenance_ops",
+ "Total number of maintenance operations",
+ Attributes.of(OPERATION_ATTR, "split"));
numErrorsCumulative =
- new AttributedLongCounter(baseErrorsMetric,
baseAttributes.toBuilder().build());
+ factory.attributedLongCounter(
+ "solr_core_update_errors", "Total number of update errors",
Attributes.empty());
+
+ submittedAdds =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "adds"));
+
+ submittedDeleteById =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
+ submittedDeleteByQuery =
+ factory.attributedLongCounter(
+ "solr_core_update_submitted_ops",
+ "Total number of submitted update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
+
+ committedAdds =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "adds"));
+
+ committedDeleteById =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_id"));
+
+ committedDeleteByQuery =
+ factory.attributedLongCounter(
+ "solr_core_update_committed_ops",
+ "Total number of committed update operations",
+ Attributes.of(OPERATION_ATTR, "deletes_by_query"));
+
+ // Create observable metrics only for core registry
observables.add(
solrMetricsContext.observableLongCounter(
"solr_core_update_auto_commits",
@@ -361,38 +396,6 @@ public class DirectUpdateHandler2 extends UpdateHandler
}));
this.toClose = Collections.unmodifiableList(observables);
-
- var baseSubmittedOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_submitted_ops", "Total number of submitted
update operations");
-
- var baseCommittedOpsMetric =
- solrMetricsContext.longCounter(
- "solr_core_update_committed_ops", "Total number of committed
update operations");
-
- submittedAdds =
- new AttributedLongCounter(
- baseSubmittedOpsMetric,
baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());
- submittedDeleteById =
- new AttributedLongCounter(
- baseSubmittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_id").build());
- submittedDeleteByQuery =
- new AttributedLongCounter(
- baseSubmittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_query").build());
-
- committedAdds =
- new AttributedLongCounter(
- baseCommittedOpsMetric,
baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build());
- committedDeleteById =
- new AttributedLongCounter(
- baseCommittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_id").build());
- committedDeleteByQuery =
- new AttributedLongCounter(
- baseCommittedOpsMetric,
- baseAttributes.toBuilder().put(OPERATION_ATTR,
"deletes_by_query").build());
}
private void deleteAll() throws IOException {
diff --git
a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
index f23456d060a..2b1586ae253 100644
---
a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
+++
b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml
@@ -29,7 +29,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
- <updateHandler class="solr.DirectUpdateHandler2">
+ <updateHandler class="solr.DirectUpdateHandler2"
aggregateNodeLevelMetricsEnabled="true">
<commitWithin>
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
</commitWithin>
@@ -52,4 +52,4 @@
<indexConfig>
<mergeScheduler
class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
</indexConfig>
-</config>
\ No newline at end of file
+</config>
diff --git
a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
index 74d3d6b2510..2320b526b08 100644
--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler;
+import static org.apache.solr.metrics.SolrMetricProducer.HANDLER_ATTR;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
@@ -196,6 +197,6 @@ public class RequestHandlerBaseTest extends SolrTestCaseJ4 {
when(metricsContext.longHistogram(any(),
any())).thenReturn(mockLongHistogram);
return new RequestHandlerBase.HandlerMetrics(
- metricsContext, Attributes.of(AttributeKey.stringKey("/handler"),
"/someBaseMetricPath"));
+ metricsContext, Attributes.of(HANDLER_ATTR, "/someBaseMetricPath"),
false);
}
}
diff --git
a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
index e9618297760..e338939b30c 100644
--- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java
@@ -16,19 +16,17 @@
*/
package org.apache.solr.handler;
+import io.prometheus.metrics.model.snapshots.CounterSnapshot;
+import io.prometheus.metrics.model.snapshots.Labels;
import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.SolrMetricTestUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -54,10 +52,7 @@ public class RequestHandlerMetricsTest extends
SolrCloudTestCase {
System.clearProperty("metricsEnabled");
}
- // TODO: Migrate aggregateNodeLevelMetricsEnabled for OTEL metrics
@Test
- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17865";)
- @SuppressWarnings({"unchecked"})
public void testAggregateNodeLevelMetrics() throws SolrServerException,
IOException {
String collection1 = "testRequestHandlerMetrics1";
String collection2 = "testRequestHandlerMetrics2";
@@ -82,97 +77,83 @@ public class RequestHandlerMetricsTest extends
SolrCloudTestCase {
cloudClient.query(collection1, solrQuery);
cloudClient.query(collection2, solrQuery);
- NamedList<Object> response =
- cloudClient.request(
- new GenericSolrRequest(
- SolrRequest.METHOD.GET, "/admin/metrics",
SolrRequest.SolrRequestType.ADMIN));
-
- NamedList<Object> metrics = (NamedList<Object>) response.get("metrics");
-
- final double[] minQueryTime = {Double.MAX_VALUE};
- final double[] maxQueryTime = {-1.0};
- final double[] minUpdateTime = {Double.MAX_VALUE};
- final double[] maxUpdateTime = {-1.0};
- Set<NamedList<Object>> coreMetrics = new HashSet<>();
- metrics.forEach(
- (key, coreMetric) -> {
- if (key.startsWith("solr.core.testRequestHandlerMetrics")) {
- coreMetrics.add((NamedList<Object>) coreMetric);
- }
- });
- assertEquals(2, coreMetrics.size());
- coreMetrics.forEach(
- metric -> {
- assertEquals(
- 1L,
- ((Map<String, Number>) metric.get("QUERY./select.requestTimes"))
- .get("count")
- .longValue());
- minQueryTime[0] =
- Math.min(
- minQueryTime[0],
- ((Map<String, Number>)
metric.get("QUERY./select.requestTimes"))
- .get("min_ms")
- .doubleValue());
- maxQueryTime[0] =
- Math.max(
- maxQueryTime[0],
- ((Map<String, Number>)
metric.get("QUERY./select.requestTimes"))
- .get("max_ms")
- .doubleValue());
- assertEquals(
- 1L,
- ((Map<String, Number>) metric.get("UPDATE./update.requestTimes"))
- .get("count")
- .longValue());
- minUpdateTime[0] =
- Math.min(
- minUpdateTime[0],
- ((Map<String, Number>)
metric.get("UPDATE./update.requestTimes"))
- .get("min_ms")
- .doubleValue());
- maxUpdateTime[0] =
- Math.max(
- maxUpdateTime[0],
- ((Map<String, Number>)
metric.get("UPDATE./update.requestTimes"))
- .get("max_ms")
- .doubleValue());
- });
-
- NamedList<Object> nodeMetrics = (NamedList<Object>)
metrics.get("solr.node");
- assertEquals(
- 2L,
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("count")
- .longValue());
- assertEquals(
- minQueryTime[0],
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("min_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- maxQueryTime[0],
- ((Map<String, Number>) nodeMetrics.get("QUERY./select.requestTimes"))
- .get("max_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- 2L,
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("count")
- .longValue());
- assertEquals(
- minUpdateTime[0],
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("min_ms")
- .doubleValue(),
- 0.0);
- assertEquals(
- maxUpdateTime[0],
- ((Map<String, Number>) nodeMetrics.get("UPDATE./update.requestTimes"))
- .get("max_ms")
- .doubleValue(),
- 0.0);
+ var coreContainer =
cluster.getJettySolrRunners().get(0).getCoreContainer();
+
+ try (SolrCore core1 =
coreContainer.getCore(coreContainer.getAllCoreNames().get(0));
+ SolrCore core2 =
coreContainer.getCore(coreContainer.getAllCoreNames().get(1))) {
+
+ CounterSnapshot.CounterDataPointSnapshot actualCore1Selects =
+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core1);
+ CounterSnapshot.CounterDataPointSnapshot actualCore1Updates =
+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core1);
+ CounterSnapshot.CounterDataPointSnapshot actualCore2Selects =
+ SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core2);
+ CounterSnapshot.CounterDataPointSnapshot actualCore2Updates =
+ SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core2);
+ CounterSnapshot.CounterDataPointSnapshot actualCore1SubmittedOps =
+ SolrMetricTestUtils.getCounterDatapoint(
+ core1,
+ "solr_core_update_submitted_ops",
+ SolrMetricTestUtils.newCloudLabelsBuilder(core1)
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot actualCore2SubmittedOps =
+ SolrMetricTestUtils.getCounterDatapoint(
+ core2,
+ "solr_core_update_submitted_ops",
+ SolrMetricTestUtils.newCloudLabelsBuilder(core2)
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .build());
+
+ assertEquals(1.0, actualCore1Selects.getValue(), 0.0);
+ assertEquals(1.0, actualCore1Updates.getValue(), 0.0);
+ assertEquals(1.0, actualCore2Updates.getValue(), 0.0);
+ assertEquals(1.0, actualCore2Selects.getValue(), 0.0);
+ assertEquals(1.0, actualCore1SubmittedOps.getValue(), 0.0);
+ assertEquals(1.0, actualCore2SubmittedOps.getValue(), 0.0);
+
+ // Get node metrics and the select/update requests should be the sum of
both cores requests
+ var nodeReader =
SolrMetricTestUtils.getPrometheusMetricReader(coreContainer, "solr.node");
+
+ CounterSnapshot.CounterDataPointSnapshot nodeSelectRequests =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_requests",
+ Labels.builder()
+ .label("category", "QUERY")
+ .label("handler", "/select")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot nodeUpdateRequests =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_requests",
+ Labels.builder()
+ .label("category", "UPDATE")
+ .label("handler", "/update")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+ CounterSnapshot.CounterDataPointSnapshot nodeSubmittedOps =
+ (CounterSnapshot.CounterDataPointSnapshot)
+ SolrMetricTestUtils.getDataPointSnapshot(
+ nodeReader,
+ "solr_node_update_submitted_ops",
+ Labels.builder()
+ .label("category", "UPDATE")
+ .label("ops", "adds")
+ .label("otel_scope_name", "org.apache.solr")
+ .build());
+
+ assertNotNull("Node select requests should be recorded",
nodeSelectRequests);
+ assertNotNull("Node update requests should be recorded",
nodeUpdateRequests);
+ assertNotNull("Node submitted update operations should be recorded",
nodeSubmittedOps);
+ assertEquals(2.0, nodeSelectRequests.getValue(), 0.0);
+ assertEquals(2.0, nodeUpdateRequests.getValue(), 0.0);
+ assertEquals(2.0, nodeSubmittedOps.getValue(), 0.0);
+ }
}
}
