STORM-1723 Introduce ClusterMetricsConsumer * ClusterMetricsConsumer publishes cluster-side related metrics into consumers * like MetricsConsumer for topology metrics * Users can implement IClusterMetricsConsumer and configure to cluster conf. file to take effect * Please refer conf/storm.yaml.example for more details on configuring * Nimbus should be launched with additional jars which are needed for IClusterMetricsConsumer * Also did some refactor to nimbus.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/622b6d13 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/622b6d13 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/622b6d13 Branch: refs/heads/1.x-branch Commit: 622b6d13594a40aa1a171c14ff660375977ea336 Parents: 9c6e6df Author: Jungtaek Lim <[email protected]> Authored: Thu Apr 21 20:40:51 2016 +0900 Committer: Jungtaek Lim <[email protected]> Committed: Sun Jun 12 21:22:09 2016 +0900 ---------------------------------------------------------------------- conf/defaults.yaml | 3 + conf/storm.yaml.example | 9 + log4j2/cluster.xml | 14 + .../src/clj/org/apache/storm/daemon/nimbus.clj | 1544 +++++++++--------- storm-core/src/jvm/org/apache/storm/Config.java | 17 +- .../metric/ClusterMetricsConsumerExecutor.java | 76 + .../metric/LoggingClusterMetricsConsumer.java | 86 + .../org/apache/storm/metric/api/DataPoint.java | 43 + .../metric/api/IClusterMetricsConsumer.java | 63 + .../storm/metric/api/IMetricsConsumer.java | 2 + .../storm/validation/ConfigValidation.java | 16 + 11 files changed, 1131 insertions(+), 742 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/conf/defaults.yaml ---------------------------------------------------------------------- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 3f6a1c9..c7a908f 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -285,3 +285,6 @@ pacemaker.kerberos.users: [] #default storm daemon metrics reporter plugins storm.daemon.metrics.reporter.plugins: - "org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter" + +# configuration of cluster metrics consumer +storm.cluster.metrics.consumer.publish.interval.secs: 60 http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/conf/storm.yaml.example ---------------------------------------------------------------------- diff --git a/conf/storm.yaml.example b/conf/storm.yaml.example index 7df3e9d..17589f1 100644 --- a/conf/storm.yaml.example +++ b/conf/storm.yaml.example @@ -46,3 +46,12 @@ # parallelism.hint: 1 # argument: # - endpoint: "metrics-collector.mycompany.org" + +## Cluster Metrics Consumers +# storm.cluster.metrics.consumer.register: +# - class: "org.apache.storm.metric.LoggingClusterMetricsConsumer" +# - class: "org.mycompany.MyMetricsConsumer" +# argument: +# - endpoint: "metrics-collector.mycompany.org" +# +# storm.cluster.metrics.consumer.publish.interval.secs: 60 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm/blob/622b6d13/log4j2/cluster.xml ---------------------------------------------------------------------- diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml index eddfae7..e911823 100644 --- a/log4j2/cluster.xml +++ b/log4j2/cluster.xml @@ -54,6 +54,17 @@ </Policies> <DefaultRolloverStrategy max="9"/> </RollingFile> + <RollingFile name="METRICS" + fileName="${sys:storm.log.dir}/${sys:logfile.name}.metrics" + filePattern="${sys:storm.log.dir}/${sys:logfile.name}.metrics.%i.gz"> + <PatternLayout> + <pattern>${patternMetrics}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="2 MB"/> + </Policies> + <DefaultRolloverStrategy max="9"/> + </RollingFile> <Syslog name="syslog" format="RFC5424" charset="UTF-8" host="localhost" port="514" protocol="UDP" appName="[${sys:daemon.name}]" mdcId="mdc" includeMDC="true" facility="LOCAL5" enterpriseNumber="18060" newLine="true" exceptionPattern="%rEx{full}" @@ -69,6 +80,9 @@ <AppenderRef ref="THRIFT-ACCESS"/> <AppenderRef ref="syslog"/> </Logger> + <Logger name="org.apache.storm.metric.LoggingClusterMetricsConsumer" level="info" additivity="false"> + <appender-ref ref="METRICS"/> + </Logger> <root level="info"> <!-- We log everything --> <appender-ref ref="A1"/> <appender-ref ref="syslog"/>
