[
https://issues.apache.org/jira/browse/EAGLE-928?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15887397#comment-15887397
]
ASF GitHub Bot commented on EAGLE-928:
--------------------------------------
Github user haoch commented on a diff in the pull request:
https://github.com/apache/eagle/pull/842#discussion_r103380011
--- Diff:
eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/builder/ApplicationBuilder.java
---
@@ -1,127 +1,131 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.app.environment.builder;
-
-
-import backtype.storm.generated.StormTopology;
-import backtype.storm.topology.TopologyBuilder;
-import backtype.storm.tuple.Fields;
-import com.google.common.base.Preconditions;
-import com.typesafe.config.Config;
-import org.apache.eagle.app.environment.impl.StormEnvironment;
-import org.apache.eagle.app.messaging.MetricSchemaGenerator;
-import org.apache.eagle.app.messaging.MetricStreamPersist;
-import org.apache.eagle.app.messaging.StormStreamSource;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Storm Application Builder DSL.
- */
-public class ApplicationBuilder {
- private final StormEnvironment environment;
- private final Config appConfig;
- private final TopologyBuilder topologyBuilder;
- private final AtomicInteger identifier;
-
- public ApplicationBuilder(Config appConfig, StormEnvironment
environment) {
- this.appConfig = appConfig;
- this.environment = environment;
- this.identifier = new AtomicInteger(0);
- this.topologyBuilder = new TopologyBuilder();
- }
-
- public class BuilderContext {
- public StormTopology toTopology() {
- return topologyBuilder.createTopology();
- }
- }
-
- public abstract class InitializedStream extends BuilderContext {
- private String id;
-
- InitializedStream(String id) {
- Preconditions.checkNotNull(id);
- this.id = id;
- }
-
- String getId() {
- return this.id;
- }
-
- /**
- * Persist source data stream as metric.
- */
- public BuilderContext saveAsMetric(MetricDefinition
metricDefinition) {
- String metricDataID = generateId("MetricDataSink");
- String metricSchemaID = generateId("MetricSchemaGenerator");
- topologyBuilder.setBolt(metricDataID, new
MetricStreamPersist(metricDefinition, appConfig)).shuffleGrouping(getId());
- topologyBuilder.setBolt(metricSchemaID, new
MetricSchemaGenerator(metricDefinition,appConfig)).fieldsGrouping(metricDataID,new
Fields(MetricStreamPersist.METRIC_NAME_FIELD));
- return this;
- }
-
- public TransformedStream transformBy(TransformFunction function) {
- String componentId = generateId(function.getName());
- topologyBuilder.setBolt(componentId, new
TransformFunctionBolt(function)).shuffleGrouping(getId());
- return new TransformedStream(componentId);
- }
- }
-
- public class SourcedStream extends InitializedStream {
- private final Config appConfig;
- private final StormStreamSource streamSource;
-
- private SourcedStream(SourcedStream withSourcedStream) {
- this(withSourcedStream.getId(), withSourcedStream.appConfig,
withSourcedStream.streamSource);
- }
-
- private SourcedStream(String componentId, Config appConfig,
StormStreamSource streamSource) {
- super(componentId);
- this.appConfig = appConfig;
- this.streamSource = streamSource;
- topologyBuilder.setSpout(componentId, streamSource);
- }
- }
-
- public class TransformedStream extends InitializedStream {
- public TransformedStream(String id) {
- super(id);
- throw new IllegalStateException("TODO: Not implemented yet");
- }
- }
-
- public TopologyBuilder getTopologyBuilder() {
- return this.topologyBuilder;
- }
-
- public StormTopology createTopology() {
- return topologyBuilder.createTopology();
- }
-
-
- public SourcedStream fromStream(String streamId) {
- return new SourcedStream(generateId("SourcedStream-" + streamId),
this.appConfig, environment.getStreamSource(streamId, this.appConfig));
- }
-
- public SourcedStream fromStream(SourcedStream sourcedStream) {
- return new SourcedStream(sourcedStream);
- }
-
- private String generateId(String prefix) {
- return String.format("%s_%s", prefix,
this.identifier.getAndIncrement());
- }
+/*
--- End diff --
dos2unix
> Refine system metric schema design and fix system metric collector
> ------------------------------------------------------------------
>
> Key: EAGLE-928
> URL: https://issues.apache.org/jira/browse/EAGLE-928
> Project: Eagle
> Issue Type: Bug
> Affects Versions: v0.5.0
> Reporter: Hao Chen
> Assignee: Hao Chen
> Fix For: v0.5.0
>
>
> Refine system metric schema design and fix system metric collector
> h1. Principle
> Any metric (no mater sys metric) should at least have three fields: "metric",
> "group", "timestamp", "value",
> "group" is the metric category path separated with slash, for example "FIRST
> CATEGORY.SECOND CATEGORY", which will be commonly used for well organizing
> metric groups in customized metric dashboard.
> h1. Stream Name for System Metric
> {code}
> <stream>
> <streamId>SYSTEM_METRIC_STREAM</streamId>
> <description>System Metrics Stream including CPU, Network, Disk,
> etc.</description>
> <columns>
> <column>
> <name>host</name>
> <type>string</type>
> </column>
> <column>
> <name>timestamp</name>
> <type>long</type>
> </column>
> <column>
> <name>metric</name>
> <type>string</type>
> </column>
> <column>
> <name>group</name>
> <type>string</type>
> </column>
> <column>
> <name>site</name>
> <type>string</type>
> </column>
> <column>
> <name>device</name>
> <type>string</type>
> </column>
> <column>
> <name>value</name>
> <type>double</type>
> <defaultValue>0.0</defaultValue>
> </column>
> </columns>
> </stream>
> {code}
> h1. Stream Schema for System metric
>
> * *metric*: [STRING] metric name string
> * *group*: [STRING] metric group/type
> * *timestamp*: [LONG] metric generation time
> * *site*: [STRING] siteId
> * *host*: [STRING] source host name
> * *device*: [STRING] device name, like cpu
> h1. Sample CPU Metric
> {code}
> {
> "timestamp": 1487918913569,
> "metric": "system.cpu.usage",
> "group": "system.cpu",
> "site": "sandbox",
> "value": 0.058,
> "host": "sandbox.hortonworks.com",
> "device": "cpu7"
> }
> {code}
> h1. Sample Network Metrics
> {code}
> {
> "timestamp": 1487918913569,
> "metric": "system.nic.transmitdrop",
> "group": "system.network",
> "site": "sandbox",
> "value": 7724.0,
> "host": "sandbox.hortonworks.com",
> "device": "eth0"
> }
> {code}
> h1. Sample Metric Schema
> {code}
> {
> "tags": {
> "site": "sandbox",
> "name": "system.memory.nfs_unstable.kb",
> "group": "system.memory"
> },
> "dimensionFields": [
> "host",
> "group",
> "site",
> "device"
> ],
> "metricFields": [
> "value"
> ],
> "granularity": "MINUTE",
> "modifiedTimestamp": 1488190388479
> }
> {code}
> h1. Stream Persist Topology
> {code}
> environment.newApp(config)
> .fromStream("HADOOP_JMX_METRIC_STREAM")
> .saveAsMetric(
> MetricDescriptor.metricGroupAs((MetricGroupSelector)
> event -> {
> if (event.containsKey("component")) {
> return String.format("hadoop.%s", ((String)
> event.get("component")).toLowerCase());
> } else {
> return "hadoop.metrics";
> }
> })
> .siteByField("site")
> .namedByField("metric")
> .eventTimeByField("timestamp")
> .dimensionFields("host", "component", "site")
> .granularity(Calendar.MINUTE)
> .valueField("value"))
> .fromStream("SYSTEM_METRIC_STREAM")
> .saveAsMetric(MetricDescriptor.metricGroupByField("group")
> .siteByField("site")
> .namedByField("metric")
> .eventTimeByField("timestamp")
> .dimensionFields("host", "group", "site", "device")
> .granularity(Calendar.MINUTE)
> .valueField("value")
> )
> .toTopology();
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
