This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2e32ec2577 Add metric for replication count from config (#8761)
2e32ec2577 is described below
commit 2e32ec25773ae664b538d306d6166fa3dbaee1b2
Author: Neha Pawar <[email protected]>
AuthorDate: Mon May 23 16:38:14 2022 -0700
Add metric for replication count from config (#8761)
---
.../configs/controller.yml | 6 ++++++
.../pinot/common/metrics/ControllerGauge.java | 1 +
.../controller/helix/SegmentStatusChecker.java | 22 ++++++++++++++++++++++
.../controller/helix/SegmentStatusCheckerTest.java | 13 +++++++++++++
4 files changed, 42 insertions(+)
diff --git
a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
index 0157c8e814..f8d116713f 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
@@ -20,6 +20,12 @@ rules:
labels:
table: "$1"
tableType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.replicationFromConfig.(\\w+)_(\\w+)\"><>(\\w+)"
+ name: "pinot_controller_replicationFromConfig_$3"
+ cache: true
+ labels:
+ table: "$1"
+ tableType: "$2"
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\",
name=\"pinot.controller.numberOfReplicas.(\\w+)_(\\w+)\"><>(\\w+)"
name: "pinot_controller_numberOfReplicas_$3"
cache: true
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index 764b8337ae..2951cfc7fb 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.Utils;
*/
public enum ControllerGauge implements AbstractMetrics.Gauge {
+ REPLICATION_FROM_CONFIG("replicas", false),
// Number of complete replicas of table in external view containing all
segments online in ideal state
NUMBER_OF_REPLICAS("replicas", false),
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index 6d14cb46df..a05dac5f9b 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -41,6 +41,7 @@ import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import
org.apache.pinot.controller.helix.core.periodictask.ControllerPeriodicTask;
import org.apache.pinot.controller.util.TableSizeReader;
+import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.slf4j.Logger;
@@ -108,6 +109,7 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
@Override
protected void processTable(String tableNameWithType, Context context) {
try {
+ updateTableConfigMetrics(tableNameWithType);
updateSegmentMetrics(tableNameWithType, context);
updateTableSizeMetrics(tableNameWithType);
} catch (Exception e) {
@@ -124,6 +126,26 @@ public class SegmentStatusChecker extends
ControllerPeriodicTask<SegmentStatusCh
_controllerMetrics.setValueOfGlobalGauge(ControllerGauge.DISABLED_TABLE_COUNT,
context._disabledTableCount);
}
+ /**
+ * Updates metrics related to the table config.
+ * If table config not found, resets the metrics
+ */
+ private void updateTableConfigMetrics(String tableNameWithType) {
+ TableConfig tableConfig =
_pinotHelixResourceManager.getTableConfig(tableNameWithType);
+ if (tableConfig == null) {
+ LOGGER.warn("Found null table config for table: {}. Resetting table
config metrics.", tableNameWithType);
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
ControllerGauge.REPLICATION_FROM_CONFIG, 0);
+ return;
+ }
+ int replication;
+ if (tableConfig.getTableType() == TableType.REALTIME) {
+ replication =
tableConfig.getValidationConfig().getReplicasPerPartitionNumber();
+ } else {
+ replication = tableConfig.getValidationConfig().getReplicationNumber();
+ }
+ _controllerMetrics.setValueOfTableGauge(tableNameWithType,
ControllerGauge.REPLICATION_FROM_CONFIG, replication);
+ }
+
private void updateTableSizeMetrics(String tableNameWithType)
throws InvalidConfigException {
_tableSizeReader.getTableSizeDetails(tableNameWithType,
TABLE_CHECKER_TIMEOUT_MS);
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
index dbcc0515d9..bdac60e617 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
@@ -43,8 +43,11 @@ import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.util.TableSizeReader;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.metrics.PinotMetricsRegistry;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -74,6 +77,9 @@ public class SegmentStatusCheckerTest {
final String tableName = "myTable_OFFLINE";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).setNumReplicas(2).build();
+
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
@@ -105,6 +111,7 @@ public class SegmentStatusCheckerTest {
{
_helixResourceManager = mock(PinotHelixResourceManager.class);
when(_helixResourceManager.getAllTables()).thenReturn(allTableNames);
+
when(_helixResourceManager.getTableConfig(tableName)).thenReturn(tableConfig);
when(_helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(_helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
}
@@ -144,6 +151,7 @@ public class SegmentStatusCheckerTest {
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
+ Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableName,
ControllerGauge.REPLICATION_FROM_CONFIG), 2);
Assert
.assertEquals(_controllerMetrics.getValueOfTableGauge(externalView.getId(),
ControllerGauge.SEGMENT_COUNT), 3);
Assert.assertEquals(
@@ -171,6 +179,9 @@ public class SegmentStatusCheckerTest {
final String rawTableName =
TableNameBuilder.extractRawTableName(tableName);
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(tableName).setTimeColumnName("timeColumn").setLLC(true)
+ .setNumReplicas(3).build();
final LLCSegmentName seg1 = new LLCSegmentName(rawTableName, 1, 0,
System.currentTimeMillis());
final LLCSegmentName seg2 = new LLCSegmentName(rawTableName, 1, 1,
System.currentTimeMillis());
final LLCSegmentName seg3 = new LLCSegmentName(rawTableName, 2, 1,
System.currentTimeMillis());
@@ -201,6 +212,7 @@ public class SegmentStatusCheckerTest {
{
_helixResourceManager = mock(PinotHelixResourceManager.class);
_helixPropertyStore = mock(ZkHelixPropertyStore.class);
+
when(_helixResourceManager.getTableConfig(tableName)).thenReturn(tableConfig);
when(_helixResourceManager.getPropertyStore()).thenReturn(_helixPropertyStore);
when(_helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(_helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
@@ -227,6 +239,7 @@ public class SegmentStatusCheckerTest {
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
+ Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableName,
ControllerGauge.REPLICATION_FROM_CONFIG), 3);
Assert.assertEquals(
_controllerMetrics.getValueOfTableGauge(externalView.getId(),
ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
Assert
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
