This is an automated email from the ASF dual-hosted git repository.
nicknezis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 01abf01 Add Prometheus metrics rules (#3651)
01abf01 is described below
commit 01abf01f123750e7fd92bc1153ac1ddaeeabe589
Author: choi se <[email protected]>
AuthorDate: Mon Dec 21 13:05:59 2020 +0900
Add Prometheus metrics rules (#3651)
---
.../config/src/yaml/conf/aurora/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/examples/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/kubernetes/metrics_sinks.yaml | 145 ++++++++++++++++++++-
.../config/src/yaml/conf/local/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/localzk/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/marathon/metrics_sinks.yaml | 131 +++++++++++++++++++
.../config/src/yaml/conf/mesos/metrics_sinks.yaml | 131 +++++++++++++++++++
.../config/src/yaml/conf/nomad/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/sandbox/metrics_sinks.yaml | 131 +++++++++++++++++++
.../config/src/yaml/conf/slurm/metrics_sinks.yaml | 131 +++++++++++++++++++
.../src/yaml/conf/test/test_metrics_sinks.yaml | 131 +++++++++++++++++++
heron/config/src/yaml/conf/yarn/metrics_sinks.yaml | 131 +++++++++++++++++++
12 files changed, 1579 insertions(+), 7 deletions(-)
diff --git a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
b/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
index 9d36164..cfcc7f0 100644
--- a/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/examples/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
b/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
index d6c5a39..081f5c7 100644
--- a/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/kubernetes/metrics_sinks.yaml
@@ -76,13 +76,144 @@ tmanager-sink:
"__time_spent_back_pressure_by_compid": SUM
prometheus-sink:
- class: "org.apache.heron.metricsmgr.sink.PrometheusSink"
- port: 8080 # The port on which to run (either port or port-file are
mandatory)
- path: /metrics # The path on which to publish the metrics (mandatory)
- flat-metrics: true # By default the web-sink will publish a flat "name ->
value" json map
- include-topology-name: true # Include topology name in metric name (default
false)
- metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
- metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+ class: "org.apache.heron.metricsmgr.sink.PrometheusSink"
+ port: 8080 # The port on which to run (either port or port-file are
mandatory)
+ path: /metrics # The path on which to publish the metrics (mandatory)
+ flat-metrics: true # By default the web-sink will publish a flat "name ->
value" json map
+ include-topology-name: true # Include topology name in metric name (default
false)
+ metrics-cache-max-size: 1000000 # Max number of metrics cached and published
(default 1000000)
+ metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+ rules:
+ # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+ - pattern: __jvm-(.+)/(.+)
+ name: jvm_$1_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ # "__execute-time-ns/pulsar-prod-4/default": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$2"
+ context: "$3"
+ # "__execute-time-ns/pulsar-prod-4": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ context: "$2"
+ # StreamManager
+ # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+ - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+ name: $2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$1"
+ # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+ - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+ name: $1_$4
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+ - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+ - pattern: PulsarSpout/(.+)/(.+)
+ name: pulsar_spout_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ - pattern: PulsarBolt/(.+)/(.+)
+ name: pulsar_bolt_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+ - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ node_id: "$4"
+ # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+ - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+ - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ partition: "$5"
+ # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+ - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+ - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ node_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+ - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+ name: kafka_offset_$3
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
+ partition: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+ - pattern: kafkaOffset/(.+)/(.+)
+ name: kafka_offset_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
### Config for metricscache-sink
metricscache-sink:
diff --git a/heron/config/src/yaml/conf/local/metrics_sinks.yaml
b/heron/config/src/yaml/conf/local/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/local/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/local/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
b/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/localzk/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
b/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/marathon/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
b/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/mesos/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
index 6a3c4fb..2e7d426 100644
--- a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
@@ -84,6 +84,137 @@ prometheus-sink:
include-topology-name: true # Include topology name in metric name (default
false)
metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+ rules:
+ # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+ - pattern: __jvm-(.+)/(.+)
+ name: jvm_$1_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ # "__execute-time-ns/pulsar-prod-4/default": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$2"
+ context: "$3"
+ # "__execute-time-ns/pulsar-prod-4": "418764",
+ - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ context: "$2"
+ # StreamManager
+ # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+ - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+ name: $2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_component: "$1"
+ # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+ - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+ name: $1_$4
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+ - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+ name: $1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ dest_container: "$2"
+ dest_task: "$3"
+ # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+ - pattern: PulsarSpout/(.+)/(.+)
+ name: pulsar_spout_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ - pattern: PulsarBolt/(.+)/(.+)
+ name: pulsar_bolt_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ component: "$1"
+ # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+ - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ node_id: "$4"
+ # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+ - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+ - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ partition: "$5"
+ # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+ - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ topic: "$4"
+ # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+ - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+ name: kafka_consumer_$2_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ node_id: "$3"
+ - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+ name: kafka_consumer_$1
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ client_id: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+ - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+ name: kafka_offset_$3
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
+ partition: "$2"
+ # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+ - pattern: kafkaOffset/(.+)/(.+)
+ name: kafka_offset_$2
+ attrNameSnakeCase: true
+ type: COUNTER
+ labels:
+ topic: "$1"
### Config for metricscache-sink
metricscache-sink:
diff --git a/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
b/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/sandbox/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
b/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/slurm/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
diff --git a/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
b/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
index 6ae89b3..21add70 100644
--- a/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/test/test_metrics_sinks.yaml
@@ -111,6 +111,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for scribe-sink
# scribe-sink:
diff --git a/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
b/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
index 6530209..6cac2f1 100644
--- a/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/yarn/metrics_sinks.yaml
@@ -114,6 +114,137 @@ metricscache-sink:
# include-topology-name: true # Include topology name in metric name
(default false)
# metrics-cache-max-size: 1000000 # Max number of metrics cached and
published (default 1000000)
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was
collected will stopped being published (default 600)
+# rules:
+# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
+# - pattern: __jvm-(.+)/(.+)
+# name: jvm_$1_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# # "__execute-time-ns/pulsar-prod-4/default": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$2"
+# context: "$3"
+# # "__execute-time-ns/pulsar-prod-4": "418764",
+# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# context: "$2"
+# # StreamManager
+# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
+# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
+# name: $2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_component: "$1"
+# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes":
"0.000000",
+# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
+# name: $1_$4
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
+# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
+# name: $1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# dest_container: "$2"
+# dest_task: "$3"
+# # PulsarSpoutMetrics of PulsarSpout
'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
+# - pattern: PulsarSpout/(.+)/(.+)
+# name: pulsar_spout_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# - pattern: PulsarBolt/(.+)/(.+)
+# name: pulsar_bolt_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# component: "$1"
+# # name:
"kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# node_id: "$4"
+# # name:
"kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
+# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# # name:
"kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# partition: "$5"
+# # name:
"kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
+# - pattern:
kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# topic: "$4"
+# # name:
"kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
+# - pattern:
kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
+# name: kafka_consumer_$2_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# node_id: "$3"
+# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
+# name: kafka_consumer_$1
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# client_id: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
+# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
+# name: kafka_offset_$3
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
+# partition: "$2"
+# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
+# - pattern: kafkaOffset/(.+)/(.+)
+# name: kafka_offset_$2
+# attrNameSnakeCase: true
+# type: COUNTER
+# labels:
+# topic: "$1"
### Config for graphite-sink
### Currently the graphite-sink is disabled
