http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java new file mode 100644 index 0000000..9879d05 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized information about liveness + * <p> + * If true liveness probes are implemented, this + * datatype can be extended to publish their details. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ApplicationLivenessInformation { + /** flag set if the cluster is at size */ + public boolean allRequestsSatisfied; + + /** number of outstanding requests: those needed to satisfy */ + public int requestsOutstanding; + + /** number of requests submitted to YARN */ + public int activeRequests; + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("ApplicationLivenessInformation{"); + sb.append("allRequestsSatisfied=").append(allRequestsSatisfied); + sb.append(", requestsOutstanding=").append(requestsOutstanding); + sb.append('}'); + return sb.toString(); + } +} + +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java new file mode 100644 index 0000000..c46a59f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.slider.api.StatusKeys; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Serializable version of component data. + * <p> + * This is sent in REST calls as a JSON object âbut is also marshalled into + * a protobuf structure. Look at {@link org.apache.slider.api.proto.RestTypeMarshalling} + * for the specifics there. + * <p> + * This means that if any fields are added here. they must be added to + * <code>src/main/proto/SliderClusterMessages.proto</code> and + * the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf} + * + * See also {@link RoleStatus#serialize()} + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + +public class ComponentInformation { + + public String name; + public int priority; + public int desired, actual, releasing; + public int placementPolicy; + public int requested; + public int failed, started, startFailed, completed, totalRequested; + public int nodeFailed, failedRecently, preempted; + public int pendingAntiAffineRequestCount; + public boolean isAARequestOutstanding; + + public String failureMessage; + public List<String> containers; + + /** + * Build the statistics map from the current data + * @return a map for use in statistics reports + */ + public Map<String, Integer> buildStatistics() { + Map<String, Integer> stats = new HashMap<>(); + stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested); + stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount); + stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed); + stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently); + stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual); + stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted); + stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested); + stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started); + stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed); + return stats; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("ComponentInformation{"); + sb.append(", name='").append(name).append('\''); + sb.append(", actual=").append(actual); + sb.append(", completed=").append(completed); + sb.append(", desired=").append(desired); + sb.append(", failed=").append(failed); + sb.append(", failureMessage='").append(failureMessage).append('\''); + sb.append(", placementPolicy=").append(placementPolicy); + sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding); + sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount); + sb.append(", priority=").append(priority); + sb.append(", releasing=").append(releasing); + sb.append(", requested=").append(requested); + sb.append(", started=").append(started); + sb.append(", startFailed=").append(startFailed); + sb.append(", totalRequested=").append(totalRequested); + sb.append(", container count='") + .append(containers == null ? 0 : containers.size()) + .append('\''); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java new file mode 100644 index 0000000..6991340 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ContainerInformation.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.hadoop.registry.client.binding.JsonSerDeser; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serializable version of component instance data + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ContainerInformation { + + public String containerId; + public String component; + public String appVersion; + public Boolean released; + public int state; + public Integer exitCode; + public String diagnostics; + public long createTime; + public long startTime; + + public String host; + public String hostURL; + public String placement; + /** + * What is the tail output from the executed process (or [] if not started + * or the log cannot be picked up + */ + public String[] output; + + @Override + public String toString() { + JsonSerDeser<ContainerInformation> serDeser = + new JsonSerDeser<>( + ContainerInformation.class); + return serDeser.toString(this); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java new file mode 100644 index 0000000..8424be2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized node entry information. Must be kept in sync with the protobuf equivalent. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class NodeEntryInformation { + + /** incrementing counter of instances that failed */ + public int failed; + + /** Counter of "failed recently" events. */ + public int failedRecently; + + /** timestamp of last use */ + public long lastUsed; + + /** Number of live nodes. */ + public int live; + + /** incrementing counter of instances that have been pre-empted. */ + public int preempted; + + /** Priority */ + public int priority; + + /** instance explicitly requested on this node */ + public int requested; + + /** number of containers being released off this node */ + public int releasing; + + /** incrementing counter of instances that failed to start */ + public int startFailed; + + /** number of starting instances */ + public int starting; + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "NodeEntryInformation{"); + sb.append("priority=").append(priority); + sb.append(", live=").append(live); + sb.append(", requested=").append(requested); + sb.append(", releasing=").append(releasing); + sb.append(", starting=").append(starting); + sb.append(", failed=").append(failed); + sb.append(", failedRecently=").append(failedRecently); + sb.append(", startFailed=").append(startFailed); + sb.append(", preempted=").append(preempted); + sb.append(", lastUsed=").append(lastUsed); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java new file mode 100644 index 0000000..4fe5b4c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Serialized node information. Must be kept in sync with the protobuf equivalent. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class NodeInformation { + + public String hostname; + public String state; + public String labels; + public String rackName; + public String httpAddress; + public String healthReport; + public long lastUpdated; + public Map<String, NodeEntryInformation> entries = new HashMap<>(); + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder( + "NodeInformation{"); + sb.append("hostname='").append(hostname).append('\''); + sb.append(", state='").append(state).append('\''); + sb.append(", labels='").append(labels).append('\''); + sb.append(", rackName='").append(rackName).append('\''); + sb.append(", httpAddress='").append(httpAddress).append('\''); + sb.append(", healthReport='").append(healthReport).append('\''); + sb.append(", lastUpdated=").append(lastUpdated); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java new file mode 100644 index 0000000..741523e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.slider.core.persist.JsonSerDeser; + +import java.util.ArrayList; +import java.util.Collection; + +public class NodeInformationList extends ArrayList<NodeInformation> { + public NodeInformationList() { + } + + public NodeInformationList(Collection<? extends NodeInformation> c) { + super(c); + } + + public NodeInformationList(int initialCapacity) { + super(initialCapacity); + } + + public static JsonSerDeser<NodeInformationList> createSerializer() { + return new JsonSerDeser<>(NodeInformationList.class); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java new file mode 100644 index 0000000..223edca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/PingInformation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * Serialized information to/from Ping operations + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PingInformation { + public long time; + public String text; + public String verb; + public String body; + + @Override + public String toString() { + + final StringBuilder sb = + new StringBuilder("PingResource{"); + sb.append("time=").append(time); + sb.append(", verb=").append(verb); + sb.append(", text='").append(text).append('\''); + sb.append(", body='").append(body).append('\''); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java new file mode 100644 index 0000000..c926600 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +/** + * Simple role statistics for state views; can be generated by RoleStatus + * instances, and aggregated for summary information. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RoleStatistics { + public long activeAA = 0L; + public long actual = 0L; + public long completed = 0L; + public long desired = 0L; + public long failed = 0L; + public long failedRecently = 0L; + public long limitsExceeded = 0L; + public long nodeFailed = 0L; + public long preempted = 0L; + public long releasing = 0L; + public long requested = 0L; + public long started = 0L; + public long startFailed = 0L; + public long totalRequested = 0L; + + /** + * Add another statistics instance + * @param that the other value + * @return this entry + */ + public RoleStatistics add(final RoleStatistics that) { + activeAA += that.activeAA; + actual += that.actual; + completed += that.completed; + desired += that.desired; + failed += that.failed; + failedRecently += that.failedRecently; + limitsExceeded += that.limitsExceeded; + nodeFailed += that.nodeFailed; + preempted += that.preempted; + releasing += that.releasing; + requested += that.requested; + started += that.started; + startFailed += that.totalRequested; + totalRequested += that.totalRequested; + return this; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java new file mode 100644 index 0000000..3b95f80 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationReport; + +/** + * Description of a slider instance + */ +public class SliderInstanceDescription { + + public final String name; + public final Path path; + public final ApplicationReport applicationReport; + + public SliderInstanceDescription(String name, + Path path, + ApplicationReport applicationReport) { + this.name = name; + this.path = path; + this.applicationReport = applicationReport; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("SliderInstanceDescription{"); + sb.append("name='").append(name).append('\''); + sb.append(", path=").append(path); + sb.append(", applicationReport: ") + .append(applicationReport == null + ? "null" + : (" id " + applicationReport.getApplicationId())); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java new file mode 100644 index 0000000..da37d11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientRegistryBinder.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; +import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants; +import org.apache.hadoop.registry.client.types.Endpoint; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.encodeForRegistry; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.convertUsername; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.getCurrentUsernameUnencoded; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath; + +/** + * Generic code to get the URLs for clients via the registry + */ +public class ClientRegistryBinder { + private static final Logger log = + LoggerFactory.getLogger(ClientRegistryBinder.class); + + private final RegistryOperations operations; + + public ClientRegistryBinder(RegistryOperations operations) { + this.operations = operations; + } + + /** + * Buld the user path -switches to the system path if the user is "". + * It also cross-converts the username to ascii via punycode + * @param username username or "" + * @return the path to the user + */ + public static String homePathForUser(String username) { + Preconditions.checkArgument(username != null, "null user"); + + // catch recursion + if (username.startsWith(RegistryConstants.PATH_USERS)) { + return username; + } + + if (username.isEmpty()) { + return RegistryConstants.PATH_SYSTEM_SERVICES; + } + + // convert username to registry name + String convertedName = convertUsername(username); + + return RegistryPathUtils.join(RegistryConstants.PATH_USERS, + encodeForRegistry(convertedName)); + } + + /** + * Get the current username, before any encoding has been applied. + * @return the current user from the kerberos identity, falling back + * to the user and/or env variables. + */ + public static String currentUsernameUnencoded() { + String env_hadoop_username = System.getenv( + RegistryInternalConstants.HADOOP_USER_NAME); + return getCurrentUsernameUnencoded(env_hadoop_username); + } + + /** + * Qualify a user. + * <ol> + * <li> <code>"~"</code> maps to user home path home</li> + * <li> <code>"~user"</code> maps to <code>/users/$user</code></li> + * <li> <code>"/"</code> maps to <code>/services/</code></li> + * </ol> + * @param user the username + * @return the base path + */ + public static String qualifyUser(String user) { + // qualify the user + String t = user.trim(); + if (t.startsWith("/")) { + // already resolved + return t; + } else if (t.equals("~")) { + // self + return currentUsernameUnencoded(); + } else if (t.startsWith("~")) { + // another user + // convert username to registry name + String convertedName = convertUsername(t.substring(1)); + + return RegistryPathUtils.join(RegistryConstants.PATH_USERS, + encodeForRegistry(convertedName)); + } else { + return "/" + t; + } + } + + /** + * Look up an external REST API + * @param user user which will be qualified as per {@link #qualifyUser(String)} + * @param serviceClass service class + * @param instance instance name + * @param api API + * @return the API, or an exception is raised. + * @throws IOException + */ + public String lookupExternalRestAPI(String user, + String serviceClass, + String instance, + String api) + throws IOException { + String qualified = qualifyUser(user); + String path = servicePath(qualified, serviceClass, instance); + String restAPI = resolveExternalRestAPI(api, path); + if (restAPI == null) { + throw new PathNotFoundException(path + " API " + api); + } + return restAPI; + } + + /** + * Resolve a service record then return an external REST API exported it. + * + * @param api API to resolve + * @param path path of the service record + * @return null if the record exists but the API is absent or it has no + * REST endpoints. + * @throws IOException resolution problems, as covered in + * {@link RegistryOperations#resolve(String)} + */ + protected String resolveExternalRestAPI(String api, String path) throws + IOException { + ServiceRecord record = operations.resolve(path); + return lookupRestAPI(record, api, true); + } + + /** + * Look up an external REST API endpoint + * @param record service record + * @param api URI of api + * @param external flag to indicate this is an external record + * @return the first endpoint of the implementation, or null if there + * is no entry for the API, implementation or it's the wrong type. + */ + public static String lookupRestAPI(ServiceRecord record, + String api, boolean external) throws InvalidRecordException { + try { + String url = null; + Endpoint endpoint = getEndpoint(record, api, external); + List<String> addresses = + RegistryTypeUtils.retrieveAddressesUriType(endpoint); + if (addresses != null && !addresses.isEmpty()) { + url = addresses.get(0); + } + return url; + } catch (InvalidRecordException e) { + log.debug("looking for API {}", api, e); + return null; + } + } + + /** + * Get an endpont by API + * @param record service record + * @param api API + * @param external flag to indicate this is an external record + * @return the endpoint or null + */ + public static Endpoint getEndpoint(ServiceRecord record, + String api, + boolean external) { + return external ? record.getExternalEndpoint(api) + : record.getInternalEndpoint(api); + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java new file mode 100644 index 0000000..c3ccb1d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.registry.client.exceptions.NoRecordException; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.NotFoundException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.registry.docstore.ConfigFormat; +import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; +import org.apache.slider.core.registry.retrieve.RegistryRetriever; + +import java.io.File; +import java.io.IOException; + +import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser; +import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath; + +public class ClientUtils { + public static ServiceRecord lookupServiceRecord(RegistryOperations rops, + String user, String name) throws IOException, SliderException { + return lookupServiceRecord(rops, user, null, name); + } + + public static ServiceRecord lookupServiceRecord(RegistryOperations rops, + String user, String type, String name) throws IOException, + SliderException { + if (StringUtils.isEmpty(user)) { + user = currentUser(); + } else { + user = RegistryPathUtils.encodeForRegistry(user); + } + if (StringUtils.isEmpty(type)) { + type = SliderKeys.APP_TYPE; + } + + String path = servicePath(user, type, name); + return resolve(rops, path); + } + + public static ServiceRecord resolve(RegistryOperations rops, String path) + throws IOException, SliderException { + try { + return rops.resolve(path); + } catch (PathNotFoundException | NoRecordException e) { + throw new NotFoundException(e.getPath().toString(), e); + } + } + + public static PublishedConfiguration getConfigFromRegistry( + RegistryOperations rops, Configuration configuration, + String configName, String appName, String user, boolean external) + throws IOException, SliderException { + ServiceRecord instance = lookupServiceRecord(rops, user, appName); + + RegistryRetriever retriever = new RegistryRetriever(configuration, instance); + PublishedConfigSet configurations = retriever.getConfigurations(external); + + PublishedConfiguration published = retriever.retrieveConfiguration( + configurations, configName, external); + return published; + } + + public static String saveOrReturnConfig(PublishedConfiguration published, + String format, File destPath, String fileName) + throws BadCommandArgumentsException, IOException { + ConfigFormat configFormat = ConfigFormat.resolve(format); + if (configFormat == null) { + throw new BadCommandArgumentsException( + "Unknown/Unsupported format %s ", format); + } + PublishedConfigurationOutputter outputter = + PublishedConfigurationOutputter.createOutputter(configFormat, + published); + boolean print = destPath == null; + if (!print) { + if (destPath.isDirectory()) { + // creating it under a directory + destPath = new File(destPath, fileName); + } + outputter.save(destPath); + return null; + } else { + return outputter.asString(); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org