[
https://issues.apache.org/jira/browse/EAGLE-526?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15483236#comment-15483236
]
ASF GitHub Bot commented on EAGLE-526:
--------------------------------------
Github user haoch commented on a diff in the pull request:
https://github.com/apache/incubator-eagle/pull/430#discussion_r78320342
--- Diff:
eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/storm/AggregationSpout.java
---
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.eagle.jpm.aggregation.storm;
+
+import org.apache.eagle.jpm.aggregation.AggregationConfig;
+import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer;
+import org.apache.eagle.jpm.aggregation.state.AggregationTimeManager;
+import org.apache.eagle.jpm.util.Utils;
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Values;
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AggregationSpout extends BaseRichSpout {
+ private static final Logger LOG =
LoggerFactory.getLogger(AggregationSpout.class);
+ private static final Long MAX_WAIT_TIME = 12 * 60 * 60000L;//12 hours
+ private static final Long MAX_SAFE_TIME = 6 * 60 * 60000L;//6 hours
+
+ private Config config;
+ MetricsAggregateContainer jobProcessTime;
+
+ private SpoutOutputCollector collector;
+ private Set<Long> processStartTime;
+ private Long lastUpdateTime;
+
+ public AggregationSpout(Config config, MetricsAggregateContainer
jobProcessTime) {
+ this.config = config;
+ this.jobProcessTime = jobProcessTime;
+ this.processStartTime = new HashSet<>();
+ }
+
+ @Override
+ public void open(Map conf, TopologyContext context, final
SpoutOutputCollector collector) {
+ AggregationConfig.getInstance(config);
+
AggregationTimeManager.instance().init(AggregationConfig.get().getZkStateConfig());
+ this.collector = collector;
+ }
+
+ @Override
+ public void nextTuple() {
+ Utils.sleep(5);
+ LOG.info("start to run");
+ try {
+ //0, check processStartTime size
+ if (this.processStartTime.size() > 0) {
+ //bolts are processing
+ return;
+ }
+
+ long currentJobTime =
System.currentTimeMillis();//this.jobProcessTime.fetchLatestJobProcessTime();
+ //1, get last updateTime;
+ lastUpdateTime =
AggregationTimeManager.instance().readLastFinishTime();
+ if (lastUpdateTime == 0L) {
+ //init state, just set to currentTime - 18 hours
+ lastUpdateTime = (currentJobTime - (MAX_SAFE_TIME +
MAX_WAIT_TIME)) / 3600000 * 3600000;
+ }
+
+ if (currentJobTime - lastUpdateTime < MAX_WAIT_TIME) {
+ LOG.info("wait for max wait time, remaining {}ms",
MAX_WAIT_TIME - currentJobTime + lastUpdateTime);
+ return;
+ }
+
+ for (Long startTime = lastUpdateTime; startTime <
lastUpdateTime + MAX_SAFE_TIME;) {
+ collector.emit(new Values(startTime), startTime);
+ this.processStartTime.add(startTime);
+ startTime +=
AggregationConfig.get().getJobExtractorConfig().aggregationDuration * 1000;
+ }
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("startTime"));
+ }
+
+ @Override
+ public void ack(Object messageId) {
+ //remove from processStartTime, if size == 0, then update
lastUpdateTime
+ Long startTime = (Long)messageId;
+ LOG.info("succeed startTime {}", startTime);
--- End diff --
debug log
> aggregation framework for mr history jobs
> -----------------------------------------
>
> Key: EAGLE-526
> URL: https://issues.apache.org/jira/browse/EAGLE-526
> Project: Eagle
> Issue Type: New Feature
> Reporter: wujinhu
> Assignee: wujinhu
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
