[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user asfgit closed the pull request at: https://github.com/apache/storm/pull/1674 ---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135829430
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
--- End diff --
`Utils.getInt` also works with a default value.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135487221
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135486529
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135487044
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135485688
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135486949
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135486437
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135486989
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135484245
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
--- End diff --
Inline & replace hosts with `Collections.singleton("host-0")`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135484798
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135486340
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135484948
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135482870
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/FaultGenerateUtils.java
---
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class FaultGenerateUtils {
+
+public static List>
getSupervisorsList(int supervisorCount, int slotCount, int[][][] faults) {
--- End diff --
This looks like not used anywhere. If then let's remove, and if not let's
replace 3-dimension int array with other data structure, given that it affects
readability.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135485132
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135484434
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135485624
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,326 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+topoMap.put(topo1.getId(), topo1);
+
+Cluster cluster = new Cluster(iNimbus, supMap, new HashMap(), config);
+Topologies topologies = new Topologies(topoMap);
+BlacklistScheduler bs = new BlacklistScheduler(new
DefaultScheduler());
+bs.prepare(config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus,
TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,
"sup-0"),
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+cluster = new Cluster(iNimbus, supMap,
TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()),
config);
+bs.schedule(topologies, cluster);
+Set hosts = new HashSet<>();
+hosts.add("host-0");
+Assert.assertEquals("blacklist", hosts,
cluster.getBlacklistedHosts());
+}
+
+@Test
+public void TestBadSlot() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135476415
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135454156
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135472915
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135466857
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java
---
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.reporters;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+public class LogReporter implements IReporter {
+private static Logger LOG = LoggerFactory.getLogger(LogReporter.class);
+
+@Override
+public void report(String message) {
+LOG.warn(message);
+}
+
+@Override
+public void reportBlacklist(String supervisor, List>> toleranceBuffer) {
+String message = "add supervisor " + supervisor + " to blacklist.
The bad slot history of supervisors is :" + toleranceBuffer;
--- End diff --
Let's just use LOG to leave a log message, with using `{}`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135462664
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135470323
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455595
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
--- End diff --
Given that master branch requires JDK 8, we could just use getOrDefault()
and reduce the code.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135462490
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135456726
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135472851
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455339
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135454219
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455679
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
--- End diff --
Same as above.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135454184
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135469938
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
--- End diff --
nit: space between { and /
---
If your project is set up for it, you can reply to this email and have your
reply appear on G
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135460607
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135465650
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/reporters/IReporter.java
---
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.reporters;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * report blacklist to alert system
+ */
+public interface IReporter {
+void report(String message);
+
+void reportBlacklist(String supervisor, List>> toleranceBuffer);
--- End diff --
If we can use `Map` instead of `HashMap` let's do that. JDK 8 may help in
this case.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135460836
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455702
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135459291
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135459962
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455693
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
--- End diff --
Same as above.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455290
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455841
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
--- End diff --
If we could get rid of duplication between this and below try statement it
would be better.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135469672
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
--- End diff --
Let's use getOrDefault().
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135464409
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135462156
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135475997
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455386
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135473281
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
+_toleranceCount = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+_resumeTime = Utils.getInt(
conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+_reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+_nimbusMonitorFreqSecs = Utils.getInt(
conf.get(Config.NIMBUS_MONITOR_FREQ_SECS));
+blacklist = new TreeMap<>();
+}
+
+@Override
+public Set getBlacklist(List>>
supervisorsWithFailures, Cluster cluster, Topologies topologies) {
+Map countMap = new HashMap();
+
+for (Map> item : supervisorsWithFailures) {
+Set supervisors = item.keySet();
+for (String supervisor : supervisors) {
+int supervisorCount = 0;
+if (countMap.containsKey(supervisor)) {
+supervisorCount = countMap.get(supervisor);
+}
+countMap.put(supervisor, supervisorCount + 1);
+}
+}
+for (Map.Entry entry : countMap.entrySet()) {
+String supervisor = entry.getKey();
+int count = entry.getValue();
+if (count >= _toleranceCount) {
+if (!blacklist.containsKey(supervisor)) {// if not in
blacklist then add it and set the resume time according to config
+LOG.info("add supervisor {} to blacklist", supervisor);
+LOG.info("supervisorsWithFailures : {}",
supervi
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135454100
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135468527
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(Map conf){
+if (conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)) {
--- End diff --
Same here: Let's use getOrDefault() every places in this method.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135455670
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
--- End diff --
Same as above.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135459457
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135463762
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user HeartSaVioR commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r135462673
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.EvictingQueue;
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+private final IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected EvictingQueue>>
badSupervisorsToleranceSlidingWindow;
+protected int windowSize;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT));
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME));
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) :
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" ;
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) :
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistSt
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95340409
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
---
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.blacklist.CircularBuffer;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+public interface IBlacklistStrategy {
+
+public void prepare(IReporter reporter, int toleranceTime, int
toleranceCount, int resumeTime, int nimbusMonitorFreqSecs);
+
+public Set getBlacklist(List>>
toleranceBuffer, Cluster cluster, Topologies topologies);
+
+public void resumeFromBlacklist();
+
+}
--- End diff --
will do so
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request: https://github.com/apache/storm/pull/1674#discussion_r95340338 --- Diff: storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java --- @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.scheduler.blacklist.strategies; + +import org.apache.storm.scheduler.Cluster; +import org.apache.storm.scheduler.Topologies; +import org.apache.storm.scheduler.blacklist.CircularBuffer; --- End diff -- removed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95340314
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceTime;
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(IReporter reporter, int toleranceTime, int
toleranceCount, int resumeTime, int nimbusMonitorFreqSecs) {
--- End diff --
May be pass conf here is a better idea, but if I do so, I will have to
parse conf again in DefaultBlacklistStrategy, and I 'll copy the code from
BlacklistScheduler. I think it's not a good idea for maintainance. Any good
idea?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95339881
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
--- End diff --
Utils.getInt is done.
but I don't think it's a good idea that I set nimbusMonitorFreqSecs to 10
by default here I think change a bit in nimbus_test.clj is better.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95339408
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
--- End diff --
Sub class of ArrayBlockingQueue passed all my unit test. I think it may
work as well.
CircularBuffer is just one possible choice for this function, and it's just
first option come into my mind. It seems like ArrayBlockingQueue is a better
choice.
Thus, the name CircularBuffer is not precise enough. what would you suggest
it should be ?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95339494
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
+
+// This is the largest capacity allowed by this implementation
+private static final int MAX_CAPACITY = 1 << 30;
+
+private int size = 0;
+private int producerIndex = 0;
+private int consumerIndex = 0;
+
+private int capacity;
+
+private Serializable[] underlying;
--- End diff --
This and all below will be solved if I change it to ArrayBlockingQueue.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95338774
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
--- End diff --
done and so as the below
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r95338650
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
--- End diff --
Should I change it to private final?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94649177
--- Diff:
storm-core/test/jvm/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
---
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.INimbus;
+import org.apache.storm.scheduler.SchedulerAssignmentImpl;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestBlacklistScheduler {
+
+private static final Logger LOG =
LoggerFactory.getLogger(TestBlacklistScheduler.class);
+
+private static int currentTime = 1468216504;
+
+@Test
+public void TestBadSupervisor() {
+INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+
+Map supMap =
TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
+
+Config config = new Config();
+config.putAll(Utils.readDefaultConfig());
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
+config.put(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
+config.put(Config.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
+
+Map topoMap = new HashMap();
+
+TopologyDetails topo1 =
TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
currentTime - 2, true);
+//TopologyDetails topo2 =
TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1,
currentTime - 8,true);
+//TopologyDetails topo3 =
TestUtilsForBlacklistScheduler.getTopology("topo-3", config, 5, 15, 1, 1,
currentTime - 16,true);
+topoMap.put(topo1.getId(), topo1);
+//topoMap.put(topo2.getId(), topo2);
+//topoMap.put(topo3.getId(), topo3);
--- End diff --
Please remove the commented out code.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94665415
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
--- End diff --
nimbusMonitorFreqSecs also needs this help too
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94661600
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
--- End diff --
the isInteger annotation does not guarantee that it is an Integer you will
get, but it guarantees that you will get a Number that can be turned into an
Integer without losing data.
```
toleranceTime = Utils.getInt(
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME));
```
This holds true for `toleranceCount` and `resumeTime` too
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94675511
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
+
+// This is the largest capacity allowed by this implementation
+private static final int MAX_CAPACITY = 1 << 30;
+
+private int size = 0;
+private int producerIndex = 0;
+private int consumerIndex = 0;
+
+private int capacity;
+
+private Serializable[] underlying;
+
+// Construct a buffer which has at least the specified capacity. If
+// the value specified is a power of two then the buffer will be
+// exactly the specified size. Otherwise the buffer will be the
+// first power of two which is greater than the specified value.
+public CircularBuffer(int capacity) {
+
+if (capacity > MAX_CAPACITY) {
+throw new IllegalArgumentException("Capacity greater than " +
+"allowed");
+}
+
+this.capacity = capacity;
+underlying = new Serializable[this.capacity];
+}
+
+// Constructor used by clone()
+private CircularBuffer(CircularBuffer oldBuffer) {
+size = oldBuffer.size;
+producerIndex = oldBuffer.producerIndex;
+consumerIndex = oldBuffer.consumerIndex;
+capacity = oldBuffer.capacity;
+//bitmask = oldBuffer.bitmask;
+underlying = new Serializable[oldBuffer.underlying.length];
+System.arraycopy(oldBuffer.underlying, 0, underlying, 0,
underlying.length);
+}
+
+private boolean isFull() {
+return size == capacity;
+}
+
+public boolean add(Serializable obj) {
--- End diff --
Shouldn't this be
```public boolean add(T obj)```
Also if the is overriding an existing method we should use the `@Override`
annotation. This is for all of the methods here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94665531
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
--- End diff --
Can we set the default to
"org.apache.storm.scheduler.blacklist.reporters.LogReporter" instead of "". It
makes it just work for the unit tests.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94678615
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
---
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.TopologyDetails;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class DefaultBlacklistStrategy implements IBlacklistStrategy {
+
+private static Logger LOG =
LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
+
+private IReporter _reporter;
+
+private int _toleranceTime;
+private int _toleranceCount;
+private int _resumeTime;
+private int _nimbusMonitorFreqSecs;
+
+private TreeMap blacklist;
+
+@Override
+public void prepare(IReporter reporter, int toleranceTime, int
toleranceCount, int resumeTime, int nimbusMonitorFreqSecs) {
--- End diff --
Could we have conf passed in here too, or not make this a plugin? I know
we don't need/use it now, but if we want this to be user pluggable we should
think about what other things people might want to do.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94677722
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
--- End diff --
I am a little confused why we need all of this. It feels like we could do
all of this with an ArrayBlockingQueue.
```
public final class CircularBuffer extends
ArrayBlockingQueue {
@Override
public boolean add(T obj) {
while (!offer(obj)) {
poll();
}
}
public List toList() {
return new ArrayList<>(this);
}
}
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94675703
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
+
+// This is the largest capacity allowed by this implementation
+private static final int MAX_CAPACITY = 1 << 30;
+
+private int size = 0;
+private int producerIndex = 0;
+private int consumerIndex = 0;
+
+private int capacity;
+
+private Serializable[] underlying;
+
+// Construct a buffer which has at least the specified capacity. If
+// the value specified is a power of two then the buffer will be
+// exactly the specified size. Otherwise the buffer will be the
+// first power of two which is greater than the specified value.
+public CircularBuffer(int capacity) {
+
+if (capacity > MAX_CAPACITY) {
+throw new IllegalArgumentException("Capacity greater than " +
+"allowed");
+}
+
+this.capacity = capacity;
+underlying = new Serializable[this.capacity];
+}
+
+// Constructor used by clone()
+private CircularBuffer(CircularBuffer oldBuffer) {
--- End diff --
This shouldn't oldBuffer be `CircularBuffer` or something like that.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94668925
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
--- End diff --
Could we drop this log message, it does not seem to really be needed any
more.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94668597
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
--- End diff --
I'm not really sure why this is package, nor why it is not final. I don't
see a reason to have it be mutable at this point.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request: https://github.com/apache/storm/pull/1674#discussion_r94678822 --- Diff: storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java --- @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.storm.scheduler.blacklist.strategies; + +import org.apache.storm.scheduler.Cluster; +import org.apache.storm.scheduler.Topologies; +import org.apache.storm.scheduler.blacklist.CircularBuffer; --- End diff -- I don't think this is used. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94668464
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExceptio
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94677987
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
--- End diff --
And some javadocs explaining what it does any why would be helpful, because
even though this is currently implemented as a CircularBuffer it has slightly
different behavior from a typical Collection.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94678786
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
---
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist.strategies;
+
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.blacklist.CircularBuffer;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+public interface IBlacklistStrategy {
+
+public void prepare(IReporter reporter, int toleranceTime, int
toleranceCount, int resumeTime, int nimbusMonitorFreqSecs);
+
+public Set getBlacklist(List>>
toleranceBuffer, Cluster cluster, Topologies topologies);
+
+public void resumeFromBlacklist();
+
+}
--- End diff --
Could we have some javadocs explaining how this is used and the life cycle
of the strategy.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94665652
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+protected int toleranceTime;
+protected int toleranceCount;
+protected int resumeTime;
+protected IReporter reporter;
+protected IBlacklistStrategy blacklistStrategy;
+
+protected int nimbusMonitorFreqSecs;
+
+protected Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+protected CircularBuffer>>
badSupervisorsTolerance;
+protected Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
--- End diff --
Can we make the default
"org.apache.storm.scheduler.blacklist.strategies.DefaultBlacklistStrategy"
instead of "" for the same reasons as above.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If y
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user revans2 commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r94675699
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/CircularBuffer.java ---
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import java.io.Serializable;
+import java.util.AbstractCollection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public final class CircularBuffer extends
AbstractCollection implements Serializable {
+
+// This is the largest capacity allowed by this implementation
+private static final int MAX_CAPACITY = 1 << 30;
+
+private int size = 0;
+private int producerIndex = 0;
+private int consumerIndex = 0;
+
+private int capacity;
+
+private Serializable[] underlying;
--- End diff --
Shouldn't this be an array of type `T`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93364833
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93267533
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93210117
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93210078
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93206374
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93206136
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93205875
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r93205840
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeException
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92872082
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92843643
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92840532
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92837439
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92840162
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92839936
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92837760
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user abellina commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r92837309
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.Cluster;
+import org.apache.storm.scheduler.IScheduler;
+import org.apache.storm.scheduler.SupervisorDetails;
+import org.apache.storm.scheduler.Topologies;
+import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+IScheduler underlyingScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>>
badSupervisorsTolerance;
+private Set blacklistHost;
+
+public BlacklistScheduler(IScheduler underlyingScheduler) {
+this.underlyingScheduler = underlyingScheduler;
+}
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+underlyingScheduler.prepare(conf);
+_conf = conf;
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)) {
+toleranceTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT))
{
+toleranceCount = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if (_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)) {
+resumeTime = (Integer)
_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String reporterClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER) : "";
+try {
+reporter = (IReporter)
Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name {}",
reporterClassName);
+throw new RuntimeException(e);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+throw new RuntimeException(e);
+}
+
+String strategyClassName =
_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY) ? (String)
_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY) : "";
+try {
+blacklistStrategy = (IBlacklistStrategy)
Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name {}",
strategyClassName);
+throw new RuntimeExcepti
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r80849475
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
+
toleranceCount=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)){
+
resumeTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String
reporterClassName=_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER)?(String)_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER):"";
+try {
+
reporter=(IReporter)Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name
{}",reporterClassName);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+}
+
+String
strategyClassName=_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY)?(String)_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY):"";
+try {
+
blacklistStrategy=(IBlacklistStrategy)Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name
{}",reporterClassName);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist strategy for
name {}", reporterClassName);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist strategy for
name {}", reporterClassName);
+}
+
+
nimbusMonitorFreqSecs=(Integer)_conf.get(Config.NIMBUS_MONITOR_FREQ_SECS);
+
blacklistStrategy.prepare(reporter,toleranceTime,toleranceCount,resumeTime,nimbusMonitorFreqSecs);
+
+toleranceBuffer=new
CircularBuffer>>(toleranceTime/nimbusMonitorFreqSecs);
+cachedSupervisors=new HashMap<>();
+
+
+}
+
+@Override
+public void schedule(Topologies topologies, Cluster cluster) {
+LOG.info("running Black List scheduler");
+Map supervisors =
cluster.getSupervisors();
+for(Map.Entry
entry:supervisors.entrySet()){
+SupervisorDetails supervisorDetails=entry.getValue();
+String supervisorName=entry.getKey();
+Set ports=supervisorDetails.getAllPorts();
+LOG.info("supervisor: "+supervisorDetails.getHost()+"
ports"+ports);
--- End diff --
SupervisorDetails is a class with all the supervisors info in it. It's not
necessary to show all the info here. But if I override toString method, I have
to put everything in it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r80839269
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
+
toleranceCount=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)){
+
resumeTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
--- End diff --
same as above
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r80839265
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
--- End diff --
same as above
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user nilday commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r80839227
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
--- End diff --
Maybe it's not necessary? [BLACKLIST_SCHEDULER_TOLERANCE_TIME ]
(https://github.com/nilday/storm/blob/blacklist-scheduler/storm-core/src/jvm/org/apache/storm/Config.java#L2164)
has Annotation of
[ isInteger]
(https://github.com/apache/storm/blob/master/storm-core/src/jvm/org/apache/storm/validation/ConfigValidationAnnotations.java#L106)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user kishorvpatil commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r78082318
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
+
toleranceCount=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)){
+
resumeTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
+String
reporterClassName=_conf.containsKey(Config.BLACKLIST_SCHEDULER_REPORTER)?(String)_conf.get(Config.BLACKLIST_SCHEDULER_REPORTER):"";
+try {
+
reporter=(IReporter)Class.forName(reporterClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist reporter for name
{}",reporterClassName);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist reporter for
name {}", reporterClassName);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist reporter for
name {}", reporterClassName);
+}
+
+String
strategyClassName=_conf.containsKey(Config.BLACKLIST_SCHEDULER_STRATEGY)?(String)_conf.get(Config.BLACKLIST_SCHEDULER_STRATEGY):"";
+try {
+
blacklistStrategy=(IBlacklistStrategy)Class.forName(strategyClassName).newInstance();
+} catch (ClassNotFoundException e) {
+LOG.error("Can't find blacklist strategy for name
{}",reporterClassName);
+} catch (InstantiationException e) {
+LOG.error("Throw InstantiationException blacklist strategy for
name {}", reporterClassName);
+} catch (IllegalAccessException e) {
+LOG.error("Throw illegalAccessException blacklist strategy for
name {}", reporterClassName);
+}
+
+
nimbusMonitorFreqSecs=(Integer)_conf.get(Config.NIMBUS_MONITOR_FREQ_SECS);
+
blacklistStrategy.prepare(reporter,toleranceTime,toleranceCount,resumeTime,nimbusMonitorFreqSecs);
+
+toleranceBuffer=new
CircularBuffer>>(toleranceTime/nimbusMonitorFreqSecs);
+cachedSupervisors=new HashMap<>();
+
+
+}
+
+@Override
+public void schedule(Topologies topologies, Cluster cluster) {
+LOG.info("running Black List scheduler");
+Map supervisors =
cluster.getSupervisors();
+for(Map.Entry
entry:supervisors.entrySet()){
+SupervisorDetails supervisorDetails=entry.getValue();
+String supervisorName=entry.getKey();
+Set ports=supervisorDetails.getAllPorts();
+LOG.info("supervisor: "+supervisorDetails.getHost()+"
ports"+ports);
--- End diff --
Would it help to override `toString` method to `SupervisorDetails` class?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
co
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user kishorvpatil commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r78081891
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
+
toleranceCount=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_RESUME_TIME)){
+
resumeTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_RESUME_TIME);
+}
--- End diff --
Should we use
[Utils.getInt](https://github.com/apache/storm/blob/master/storm-core/src/jvm/org/apache/storm/utils/Utils.java#L753)
or
[Utils.getInt](https://github.com/apache/storm/blob/master/storm-core/src/jvm/org/apache/storm/utils/Utils.java#L792)
to avoid `Null` being assigned.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] storm pull request #1674: STORM-2083: Blacklist scheduler
Github user kishorvpatil commented on a diff in the pull request:
https://github.com/apache/storm/pull/1674#discussion_r78081868
--- Diff:
storm-core/src/jvm/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
---
@@ -0,0 +1,212 @@
+package org.apache.storm.scheduler.blacklist;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.*;
+import org.apache.storm.scheduler.blacklist.reporters.IReporter;
+import org.apache.storm.scheduler.blacklist.strategies.IBlacklistStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Created by howard.li on 2016/6/29.
+ */
+public class BlacklistScheduler implements IScheduler {
+private static final Logger LOG =
LoggerFactory.getLogger(BlacklistScheduler.class);
+DefaultScheduler defaultScheduler;
+@SuppressWarnings("rawtypes")
+private Map _conf;
+
+private int toleranceTime;
+private int toleranceCount;
+private int resumeTime;
+private IReporter reporter;
+private IBlacklistStrategy blacklistStrategy;
+
+private int nimbusMonitorFreqSecs;
+
+
+private Map> cachedSupervisors;
+
+//key is supervisor key ,value is supervisor ports
+private CircularBuffer>> toleranceBuffer;
+
+@Override
+public void prepare(Map conf) {
+LOG.info("prepare black list scheduler");
+LOG.info(conf.toString());
+defaultScheduler=new DefaultScheduler();
+defaultScheduler.prepare(conf);
+_conf=conf;
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME)){
+
toleranceTime=(Integer)_conf.get(Config.BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+}
+if(_conf.containsKey(Config.BLACKLIST_SCHEDULER_TOLERANCE_COUNT)){
--- End diff --
Should we use
[Utils.getInt](https://github.com/apache/storm/blob/master/storm-core/src/jvm/org/apache/storm/utils/Utils.java#L753)
or
[Utils.getInt](https://github.com/apache/storm/blob/master/storm-core/src/jvm/org/apache/storm/utils/Utils.java#L792)
to avoid `Null` being assigned.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
