http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java new file mode 100644 index 0000000..37a8935 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.MetricSet; +import com.codahale.metrics.health.HealthCheckRegistry; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Class for all metrics and monitoring + */ +public class MetricsAndMonitoring extends CompositeService { + protected static final Logger log = + LoggerFactory.getLogger(MetricsAndMonitoring.class); + public MetricsAndMonitoring(String name) { + super(name); + } + + public MetricsAndMonitoring() { + super("MetricsAndMonitoring"); + } + + /** + * Singleton of metrics registry + */ + final MetricRegistry metrics = new MetricRegistry(); + + final HealthCheckRegistry health = new HealthCheckRegistry(); + + private final Map<String, MeterAndCounter> meterAndCounterMap + = new ConcurrentHashMap<>(); + + private final List<MetricSet> metricSets = new ArrayList<>(); + + /** + * List of recorded events + */ + private final List<RecordedEvent> eventHistory = new ArrayList<>(100); + + public static final int EVENT_LIMIT = 1000; + + public MetricRegistry getMetrics() { + return metrics; + } + + public HealthCheckRegistry getHealth() { + return health; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + addService(new MetricsBindingService("MetricsBindingService", + metrics)); + super.serviceInit(conf); + } + + @Override + protected void serviceStop() throws Exception { + super.serviceStop(); + for (MetricSet set : metricSets) { + unregister(set); + } + } + + public MeterAndCounter getMeterAndCounter(String name) { + return meterAndCounterMap.get(name); + } + + /** + * Get or create the meter/counter pair + * @param name name of instance + * @return an instance + */ + public MeterAndCounter getOrCreateMeterAndCounter(String name) { + MeterAndCounter instance = meterAndCounterMap.get(name); + if (instance == null) { + synchronized (this) { + // check in a sync block + instance = meterAndCounterMap.get(name); + if (instance == null) { + instance = new MeterAndCounter(metrics, name); + meterAndCounterMap.put(name, instance); + } + } + } + return instance; + } + + /** + * Get a specific meter and mark it. This will create and register it on demand. + * @param name name of meter/counter + */ + public void markMeterAndCounter(String name) { + MeterAndCounter meter = getOrCreateMeterAndCounter(name); + meter.mark(); + } + + /** + * Given a {@link Metric}, registers it under the given name. + * + * @param name the name of the metric + * @param metric the metric + * @param <T> the type of the metric + * @return {@code metric} + * @throws IllegalArgumentException if the name is already registered + */ + public <T extends Metric> T register(String name, T metric) throws IllegalArgumentException { + return metrics.register(name, metric); + } + + public <T extends Metric> T register(Class<?> klass, T metric, String... names) + throws IllegalArgumentException { + return register(MetricRegistry.name(klass, names), metric); + } + + + /** + * Add an event (synchronized) + * @param event event + */ + public synchronized void noteEvent(RecordedEvent event) { + if (eventHistory.size() > EVENT_LIMIT) { + eventHistory.remove(0); + } + eventHistory.add(event); + } + + /** + * Clone the event history; blocks for the duration of the copy operation. + * @return a new list + */ + public synchronized List<RecordedEvent> cloneEventHistory() { + return new ArrayList<>(eventHistory); + } + + /** + * Add a metric set for registering and deregistration on service stop + * @param metricSet metric set + */ + public void addMetricSet(MetricSet metricSet) { + metricSets.add(metricSet); + metrics.registerAll(metricSet); + } + + /** + * add a metric set, giving each entry a prefix + * @param prefix prefix (a trailing "." is automatically added) + * @param metricSet the metric set to register + */ + public void addMetricSet(String prefix, MetricSet metricSet) { + addMetricSet(new PrefixedMetricsSet(prefix, metricSet)); + } + + /** + * Unregister a metric set; robust + * @param metricSet metric set to unregister + */ + public void unregister(MetricSet metricSet) { + for (String s : metricSet.getMetrics().keySet()) { + try { + metrics.remove(s); + } catch (IllegalArgumentException e) { + // log but continue + log.info("Exception when trying to unregister {}", s, e); + } + } + } +} +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java new file mode 100644 index 0000000..864a1cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.MetricSet; +import com.codahale.metrics.ScheduledReporter; +import com.codahale.metrics.Slf4jReporter; +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; +import org.apache.slider.server.services.workflow.ClosingService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * YARN service which hooks up Codahale metrics to + * JMX, and, if enabled Ganglia and/or an SLF4J log. + */ +public class MetricsBindingService extends CompositeService + implements MetricsKeys { + protected static final Logger log = + LoggerFactory.getLogger(MetricsBindingService.class); + private final MetricRegistry metrics; + + private String reportingDetails = "not started"; + + + public MetricsBindingService(String name, + MetricRegistry metrics) { + super(name); + Preconditions.checkArgument(metrics != null, "Null metrics"); + this.metrics = metrics; + } + + /** + * Instantiate...create a metric registry in the process + * @param name service name + */ + public MetricsBindingService(String name) { + this(name, new MetricRegistry()); + } + + /** + * Accessor for the metrics instance + * @return the metrics + */ + public MetricRegistry getMetrics() { + return metrics; + } + + @Override + protected void serviceStart() throws Exception { + super.serviceStart(); + + StringBuilder summary = new StringBuilder(); + Configuration conf = getConfig(); + + summary.append("Reporting to JMX"); + // always start the JMX binding + JmxReporter jmxReporter; + jmxReporter = JmxReporter.forRegistry(metrics).build(); + jmxReporter.start(); + addService(new ClosingService<>(jmxReporter)); + + + // Ganglia + if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) { + log.warn("Ganglia integration is not implemented"); +/* + // This is all disabled due to transitive dependencies on an LGPL library + com.codahale.metrics.ganglia.GangliaReporter gangliaReporter; + String host = conf.getTrimmed(METRICS_GANGLIA_HOST, ""); + int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT); + int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60); + int ttl = 1; + info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode + mcast = info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host); + boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true); + + final info.ganglia.gmetric4j.gmetric.GMetric ganglia = + new info.ganglia.gmetric4j.gmetric.GMetric( + host, + port, + mcast, + ttl, + ganglia31); + gangliaReporter = com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo( + TimeUnit.MILLISECONDS) + .build(ganglia); + gangliaReporter.start(interval, TimeUnit.SECONDS); + addService(new ClosingService<ScheduledReporter>(gangliaReporter)); + summary.append(String.format(", Ganglia at %s:%d interval=%d", + host, port, interval)); + */ + } + + // Logging + if (conf.getBoolean(METRICS_LOGGING_ENABLED, false)) { + ScheduledReporter reporter; + String logName = + conf.getTrimmed(METRICS_LOGGING_LOG, METRICS_DEFAULT_LOG); + int interval = conf.getInt(METRICS_LOGGING_LOG_INTERVAL, + METRICS_DEFAULT_LOG_INTERVAL); + reporter = Slf4jReporter.forRegistry(metrics) + .convertRatesTo(TimeUnit.SECONDS) + .outputTo(LoggerFactory.getLogger(logName)) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + reporter.start(interval, TimeUnit.MINUTES); + addService(new ClosingService<>(reporter)); + summary.append(String.format(", SLF4J to log %s interval=%d", + logName, interval)); + } + reportingDetails = summary.toString(); + log.info(reportingDetails); + } + + + @Override + public String toString() { + return super.toString() + " " + reportingDetails; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java new file mode 100644 index 0000000..fa6bfc0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +/** + * Constants used in slider for metrics registration and lookup + */ +public class MetricsConstants { + + /** + * {@value} + */ + public static final String CONTAINERS_OUTSTANDING_REQUESTS = "containers.outstanding-requests"; + + /** + * {@value} + */ + public static final String CONTAINERS_STARTED = "containers.started"; + + /** + * {@value} + */ + public static final String CONTAINERS_SURPLUS = "containers.surplus"; + + /** + * {@value} + */ + public static final String CONTAINERS_COMPLETED = "containers.completed"; + + /** + * {@value} + */ + public static final String CONTAINERS_FAILED = "containers.failed"; + + /** + * {@value} + */ + public static final String CONTAINERS_START_FAILED = "containers.start-failed"; + + public static final String PREFIX_SLIDER_ROLES = "slider.roles."; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java new file mode 100644 index 0000000..13b3b6b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsKeys.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +public interface MetricsKeys { + + /** + * Prefix for metrics configuration options: {@value} + */ + String METRICS_PREFIX = "slider.metrics."; + + /** + * Boolean to enable Ganglia metrics reporting + * {@value} + */ + String METRICS_GANGLIA_ENABLED = + METRICS_PREFIX + "ganglia.enabled"; + /** + * {@value} + */ + String METRICS_GANGLIA_HOST = METRICS_PREFIX + "ganglia.host"; + /** + * {@value} + */ + String METRICS_GANGLIA_PORT = METRICS_PREFIX + "ganglia.port"; + /** + * {@value} + */ + String METRICS_GANGLIA_VERSION_31 = METRICS_PREFIX + "ganglia.version-31"; + /** + * {@value} + */ + String METRICS_GANGLIA_REPORT_INTERVAL = METRICS_PREFIX + "ganglia.report.interval"; + /** + * {@value} + */ + int DEFAULT_GANGLIA_PORT = 8649; + + + /** + * Boolean to enable Logging metrics reporting + * {@value} + */ + String METRICS_LOGGING_ENABLED = + METRICS_PREFIX + "logging.enabled"; + + /** + * String name of log to log to + * {@value} + */ + String METRICS_LOGGING_LOG = + METRICS_PREFIX + "logging.log.name"; + + /** + * Default log name: {@value} + */ + String METRICS_DEFAULT_LOG = + "org.apache.slider.metrics.log"; + + + /** + * Int log interval in seconds + * {@value} + */ + String METRICS_LOGGING_LOG_INTERVAL = + METRICS_PREFIX + "logging.interval.minutes"; + + + /** + * Default log interval: {@value}. + * This is a big interval as in a long lived service, log overflows are easy + * to create. + */ + int METRICS_DEFAULT_LOG_INTERVAL = 60; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java new file mode 100644 index 0000000..e9ad46a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricSet; + +import java.util.HashMap; +import java.util.Map; + +/** + * From an existing metrics set, generate a new metrics set with the + * prefix in front of every key. + * + * The prefix is added directly: if you want a '.' between prefix and metric + * keys, include it in the prefix. + */ +public class PrefixedMetricsSet implements MetricSet { + + private final String prefix; + private final MetricSet source; + + public PrefixedMetricsSet(String prefix, MetricSet source) { + this.prefix = prefix; + this.source = source; + } + + @Override + public Map<String, Metric> getMetrics() { + Map<String, Metric> sourceMetrics = source.getMetrics(); + Map<String, Metric> metrics = new HashMap<>(sourceMetrics.size()); + for (Map.Entry<String, Metric> entry : sourceMetrics.entrySet()) { + metrics.put(prefix + "." + entry.getKey(), entry.getValue()); + } + return metrics; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java new file mode 100644 index 0000000..80e88fc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Counting; +import com.codahale.metrics.Metric; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * This is a counter whose range can be given a min and a max + */ +public class RangeLimitedCounter implements Metric, Counting { + + private final AtomicLong value; + private final long min, max; + + /** + * Instantiate + * @param val current value + * @param min minimum value + * @param max max value (or 0 for no max) + */ + public RangeLimitedCounter(long val, long min, long max) { + this.value = new AtomicLong(val); + this.min = min; + this.max = max; + } + + /** + * Set to a new value. If below the min, set to the minimum. If the max is non + * zero and the value is above that maximum, set it to the maximum instead. + * @param val value + */ + public synchronized void set(long val) { + if (val < min) { + val = min; + } else if (max > 0 && val > max) { + val = max; + } + value.set(val); + } + + public void inc() { + inc(1); + } + + public void dec() { + dec(1); + } + + public synchronized void inc(int delta) { + set(value.get() + delta); + } + + public synchronized void dec(int delta) { + set(value.get() - delta); + } + + public long get() { + return value.get(); + } + + @Override + public long getCount() { + return value.get(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java new file mode 100644 index 0000000..d48d337 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.text.DateFormat; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class RecordedEvent { + private static final DateFormat dateFormat = DateFormat.getDateInstance(); + public long id; + public String name; + public long timestamp; + public String time; + public String category; + public String host; + public int role; + public String text; + + public RecordedEvent() { + } + + /** + * Create an event. The timestamp is also converted to a time string + * @param id id counter + * @param name event name + * @param timestamp timestamp. If non-zero, is used to build the {@code time} text field. + * @param category even category + * @param text arbitrary text + */ + public RecordedEvent(long id, String name, long timestamp, String category, String text) { + this.id = id; + this.name = name; + this.timestamp = timestamp; + this.time = timestamp > 0 ? dateFormat.format(timestamp) : ""; + this.category = category; + this.text = text; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java new file mode 100644 index 0000000..c30e749 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +/** + * A timestamp metric + */ +public class Timestamp extends LongGauge { + + public Timestamp(long val) { + super(val); + } + + public Timestamp() { + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java new file mode 100644 index 0000000..936563c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/YarnServiceHealthCheck.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.management; + +import com.codahale.metrics.health.HealthCheck; +import org.apache.hadoop.service.Service; + +public class YarnServiceHealthCheck extends HealthCheck { + + private final Service service; + + public YarnServiceHealthCheck(Service service) { + this.service = service; + } + + @Override + protected Result check() throws Exception { + return service.isInState(Service.STATE.STARTED) + ? Result.healthy() + : Result.unhealthy("Service is not running: %s", service); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java new file mode 100644 index 0000000..a397e19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.slider.api.InternalKeys; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Entry in the chaos list + */ +public class ChaosEntry { + + protected static final Logger log = + LoggerFactory.getLogger(ChaosEntry.class); + public final String name; + public final ChaosTarget target; + public final long probability; + + private final Counter invocationCounter; + + + /** + * Constructor -includes validation of all arguments + * @param name entry name + * @param target target + * @param probability probability of occurring + */ + public ChaosEntry(String name, ChaosTarget target, long probability, + MetricRegistry metrics) { + Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name"); + Preconditions.checkArgument(target != null, "null target"); + Preconditions.checkArgument(probability > 0, "negative probability"); + Preconditions.checkArgument(probability <= InternalKeys.PROBABILITY_PERCENT_100, + "probability over 100%: "+ probability); + this.name = name; + this.target = target; + this.probability = probability; + invocationCounter = + metrics.counter(MetricRegistry.name(ChaosEntry.class, name)); + } + + /** + * Trigger the chaos action + */ + public void invokeChaos() { + log.info("Invoking {}", name); + invocationCounter.inc(); + target.chaosAction(); + } + + /** + * Invoke Chaos if the trigger value is in range of the probability + * @param value trigger value, 0-10K + * @return true if the chaos method was invoked + */ + public boolean maybeInvokeChaos(long value) { + log.debug("Probability {} trigger={}", probability, value); + if (value < probability) { + invokeChaos(); + return true; + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java new file mode 100644 index 0000000..3c1a914 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +import org.apache.slider.server.appmaster.actions.ActionHalt; +import org.apache.slider.server.appmaster.actions.QueueAccess; + +import java.util.concurrent.TimeUnit; + +/** + * Kill the AM + */ +public class ChaosKillAM implements ChaosTarget { + + public static final int DELAY = 1000; + private final QueueAccess queues; + private final int exitCode; + + public ChaosKillAM(QueueAccess queues, int exitCode) { + this.queues = queues; + this.exitCode = exitCode; + } + + /** + * Trigger a delayed halt + */ + @Override + public void chaosAction() { + queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", DELAY, + TimeUnit.MILLISECONDS)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java new file mode 100644 index 0000000..ae38e4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +import com.google.common.base.Preconditions; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.server.appmaster.actions.ActionKillContainer; +import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.slider.server.appmaster.operations.RMOperationHandler; +import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.ListIterator; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Kill a container + */ +public class ChaosKillContainer implements ChaosTarget { + protected static final Logger log = + LoggerFactory.getLogger(ChaosKillContainer.class); + public static final int DELAY = 100; + private final AppState appState; + private final QueueAccess queues; + private final Random random = new Random(); + private final RMOperationHandler operationHandler; + + public ChaosKillContainer(AppState appState, + QueueAccess queues, + RMOperationHandler operationHandler) { + Preconditions.checkNotNull(appState); + Preconditions.checkNotNull(queues); + this.appState = appState; + this.queues = queues; + this.operationHandler = operationHandler; + } + + /** + * Trigger a container kill + */ + @Override + public void chaosAction() { + List<RoleInstance> liveContainers = + appState.cloneLiveContainerInfoList(); + // purge any and all components which declare that they are an AM + ListIterator<RoleInstance> containers = + liveContainers.listIterator(); + while (containers.hasNext()) { + RoleInstance instance = containers.next(); + if (SliderKeys.COMPONENT_AM.equals(instance.role)) { + containers.remove(); + } + } + int size = liveContainers.size(); + if (size > 0) { + int target = random.nextInt(size); + RoleInstance roleInstance = liveContainers.get(target); + log.info("Killing {}", roleInstance); + + queues.schedule(new ActionKillContainer(roleInstance.getId(), + DELAY, TimeUnit.MILLISECONDS, operationHandler)); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java new file mode 100644 index 0000000..8948f0d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +import com.codahale.metrics.MetricRegistry; +import org.apache.hadoop.service.AbstractService; +import org.apache.slider.api.InternalKeys; +import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.slider.server.appmaster.actions.RenewingAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * A chaos monkey service which will invoke ChaosTarget events + */ +public class ChaosMonkeyService extends AbstractService { + protected static final Logger log = + LoggerFactory.getLogger(ChaosMonkeyService.class); + + private final MetricRegistry metrics; + private final QueueAccess queues; + private final Random random = new Random(); + + private final List<ChaosEntry> chaosEntries = + new ArrayList<ChaosEntry>(); + + public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) { + super("ChaosMonkeyService"); + this.metrics = metrics; + this.queues = queues; + } + + /** + * Add a target ... it is only added if <code>probability > 0</code> + * @param name name + * @param target chaos target + * @param probability probability + */ + public synchronized void addTarget(String name, + ChaosTarget target, long probability) { + if (probability > 0) { + log.info("Adding {} with probability {}", name, + ((double)probability) / InternalKeys.PROBABILITY_PERCENT_1); + chaosEntries.add(new ChaosEntry(name, target, probability, metrics)); + } else { + log.debug("Action {} not enabled", name); + } + } + + /** + * Get the number of targets in the list + * @return the count of added targets + */ + public int getTargetCount() { + return chaosEntries.size(); + } + + /** + * Iterate through all the entries and invoke chaos on those wanted + */ + public void play() { + for (ChaosEntry chaosEntry : chaosEntries) { + long p = randomPercentage(); + chaosEntry.maybeInvokeChaos(p); + } + } + + public int randomPercentage() { + return random.nextInt(InternalKeys.PROBABILITY_PERCENT_100); + } + + /** + * Check for callers to see if chaos should be triggered; shares the + * same random number source as the rest of the monkey entries + * @param probability probability + * @return true if the action should happen + */ + public boolean chaosCheck(long probability) { + return randomPercentage() < probability; + } + + /** + * Schedule the monkey + * + * @param delay initial delay + * @param timeUnit time unit + * @return true if it was scheduled (i.e. 1+ action) and interval > 0 + */ + public boolean schedule(long delay, long interval, TimeUnit timeUnit) { + if (interval > 0 && !chaosEntries.isEmpty()) { + queues.schedule(getChaosAction(delay, interval, timeUnit)); + return true; + } else { + return false; + } + } + + /** + * Get the chaos action + * + * @param delay + * @param timeUnit time unit + * @return the action to schedule + */ + public RenewingAction<MonkeyPlayAction> getChaosAction(long delay, + long interval, + TimeUnit timeUnit) { + RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>( + new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS), + delay, + interval, + timeUnit, + 0 + ); + return action; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java new file mode 100644 index 0000000..1c3a9ac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +public interface ChaosTarget { + + public void chaosAction(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java new file mode 100644 index 0000000..20e4466 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.monkey; + +import org.apache.slider.server.appmaster.SliderAppMaster; +import org.apache.slider.server.appmaster.actions.AsyncAction; +import org.apache.slider.server.appmaster.actions.QueueAccess; +import org.apache.slider.server.appmaster.state.AppState; + +import java.util.concurrent.TimeUnit; + +/** + * Queueable action which calls {@link ChaosMonkeyService#play()} when + * executed. + */ +public class MonkeyPlayAction extends AsyncAction { + + private final ChaosMonkeyService monkey; + + public MonkeyPlayAction(ChaosMonkeyService monkey, long delay, + TimeUnit timeUnit) { + super("chaos monkey", delay, timeUnit); + this.monkey = monkey; + } + + @Override + public void execute(SliderAppMaster appMaster, + QueueAccess queueService, + AppState appState) throws Exception { + monkey.play(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java new file mode 100644 index 0000000..ed3f197 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +public abstract class AbstractRMOperation { + + /** + * Execute the operation + * @param asyncRMClient client + * @param handler handler to perform the execution + */ + public abstract void execute(RMOperationHandlerActions handler); + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java new file mode 100644 index 0000000..03231ef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; + +/** + * Hands off RM operations to the Resource Manager. + */ +public class AsyncRMOperationHandler extends RMOperationHandler { + protected static final Logger log = + LoggerFactory.getLogger(AsyncRMOperationHandler.class); + private final AMRMClientAsync client; + private final Resource maxResources; + + public AsyncRMOperationHandler(AMRMClientAsync client, Resource maxResources) { + this.client = client; + this.maxResources = maxResources; + } + + @Override + public int cancelContainerRequests(Priority priority1, + Priority priority2, + int count) { + // need to revoke a previously issued container request + // so enum the sets and pick some + int remaining = cancelSinglePriorityRequests(priority1, count); + if (priority2 != null) { + remaining = cancelSinglePriorityRequests(priority2, remaining); + } + + return remaining; + } + + /** + * Cancel just one of the priority levels + * @param priority priority to cancel + * @param count count to cancel + * @return number of requests cancelled + */ + @SuppressWarnings("unchecked") + protected int cancelSinglePriorityRequests(Priority priority, + int count) { + List<Collection<AMRMClient.ContainerRequest>> requestSets = + client.getMatchingRequests(priority, "", maxResources); + if (count <= 0) { + return 0; + } + int remaining = count; + for (Collection<AMRMClient.ContainerRequest> requestSet : requestSets) { + if (remaining == 0) { + break; + } + for (AMRMClient.ContainerRequest request : requestSet) { + if (remaining == 0) { + break; + } + // a single release + cancelSingleRequest(request); + remaining --; + } + } + return remaining; + } + + @Override + @SuppressWarnings("unchecked") + public void cancelSingleRequest(AMRMClient.ContainerRequest request) { + // a single release + client.removeContainerRequest(request); + } + + @Override + public void releaseAssignedContainer(ContainerId containerId) { + log.debug("Releasing container {}", containerId); + + client.releaseAssignedContainer(containerId); + } + + @Override + @SuppressWarnings("unchecked") + public void addContainerRequest(AMRMClient.ContainerRequest req) { + client.addContainerRequest(req); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java new file mode 100644 index 0000000..d7673d3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.server.appmaster.state.ContainerPriority; + +/** + * Cancel a container request + */ +public class CancelSingleRequest extends AbstractRMOperation { + + private final AMRMClient.ContainerRequest request; + + public CancelSingleRequest(AMRMClient.ContainerRequest request) { + Preconditions.checkArgument(request != null, "Null container request"); + this.request = request; + } + + @Override + public void execute(RMOperationHandlerActions handler) { + handler.cancelSingleRequest(request); + } + + public AMRMClient.ContainerRequest getRequest() { + return request; + } + + @Override + public String toString() { + return "Cancel container request" + + " for :" + ContainerPriority.toString(request.getPriority()) + + " request " + request; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java new file mode 100644 index 0000000..4271d50 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.slider.server.appmaster.state.ContainerPriority; + +public class ContainerReleaseOperation extends AbstractRMOperation { + + private final ContainerId containerId; + + public ContainerReleaseOperation(ContainerId containerId) { + Preconditions.checkArgument(containerId != null, "Null containerId"); + this.containerId = containerId; + } + + public ContainerId getContainerId() { + return containerId; + } + + @Override + public void execute(RMOperationHandlerActions handler) { + handler.releaseAssignedContainer(containerId); + } + + @Override + public String toString() { + return "release container " + containerId; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java new file mode 100644 index 0000000..e29ddd0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.server.appmaster.state.ContainerPriority; + +/** + * A container request operation + */ +public class ContainerRequestOperation extends AbstractRMOperation { + + private final AMRMClient.ContainerRequest request; + + public ContainerRequestOperation(AMRMClient.ContainerRequest request) { + Preconditions.checkArgument(request != null, "Null container request"); + this.request = request; + } + + public AMRMClient.ContainerRequest getRequest() { + return request; + } + + public Priority getPriority() { + return request.getPriority(); + } + + public boolean getRelaxLocality() { + return request.getRelaxLocality(); + } + + @Override + public void execute(RMOperationHandlerActions handler) { + handler.addContainerRequest(request); + } + + @Override + public String toString() { + return "request container for role " + + ContainerPriority.toString(getPriority()) + + " request " + request + + " relaxLocality=" + getRelaxLocality(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java new file mode 100644 index 0000000..184a36a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.providers.ProviderService; + +public class ProviderNotifyingOperationHandler extends RMOperationHandler { + + private final ProviderService providerService; + + public ProviderNotifyingOperationHandler(ProviderService providerService) { + this.providerService = providerService; + } + + @Override + public void releaseAssignedContainer(ContainerId containerId) { + providerService.releaseAssignedContainer(containerId); + } + + @Override + public void addContainerRequest(AMRMClient.ContainerRequest req) { + providerService.addContainerRequest(req); + } + + @Override + public int cancelContainerRequests(Priority priority1, + Priority priority2, + int count) { + return providerService.cancelContainerRequests(priority1, priority2, count); + } + + @Override + public void cancelSingleRequest(AMRMClient.ContainerRequest request) { + providerService.cancelSingleRequest(request); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java new file mode 100644 index 0000000..d0d038a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import java.util.List; + +public abstract class RMOperationHandler implements RMOperationHandlerActions { + + @Override + public void execute(List<AbstractRMOperation> operations) { + for (AbstractRMOperation operation : operations) { + operation.execute(this); + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java new file mode 100644 index 0000000..b7794ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.operations; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.client.api.AMRMClient; + +import java.util.List; + +public interface RMOperationHandlerActions { + + /** + * Release an assigned container + * @param containerId container + */ + void releaseAssignedContainer(ContainerId containerId); + + /** + * Issue a container request + * @param request + */ + void addContainerRequest(AMRMClient.ContainerRequest request); + + /** + * Cancel a specific request + * @param request request to cancel + */ + void cancelSingleRequest(AMRMClient.ContainerRequest request); + + /** + * Remove a container request + * @param priority1 priority to remove at + * @param priority2 second priority to target + * @param count number to remove + */ + int cancelContainerRequests(Priority priority1, Priority priority2, int count); + + /** + * Execute an entire list of operations + * @param operations ops + */ + void execute(List<AbstractRMOperation> operations); +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org