SLIDER-947 build node map from yarn update reports; serve via REST/IPC. API done; now trying to make sure RM notifies AM of state, which is being checked via new metrics
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3aeab9ca Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3aeab9ca Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3aeab9ca Branch: refs/heads/develop Commit: 3aeab9ca35e32d927f5fb4b52eb2157ab537f84f Parents: de040cb Author: Steve Loughran <[email protected]> Authored: Fri Oct 16 18:53:27 2015 +0100 Committer: Steve Loughran <[email protected]> Committed: Fri Oct 16 18:53:27 2015 +0100 ---------------------------------------------------------------------- pom.xml | 14 +- slider-core/pom.xml | 6 +- .../apache/slider/api/SliderApplicationApi.java | 28 +- .../slider/api/SliderClusterProtocol.java | 25 +- .../org/apache/slider/api/proto/Messages.java | 2390 +++++++++++++++++- .../slider/api/proto/RestTypeMarshalling.java | 90 +- .../slider/api/proto/SliderClusterAPI.java | 274 +- .../slider/api/types/NodeEntryInformation.java | 41 +- .../slider/api/types/NodeInformation.java | 8 +- .../apache/slider/client/SliderClientAPI.java | 4 +- .../client/ipc/SliderApplicationIpcClient.java | 22 +- .../client/ipc/SliderClusterOperations.java | 118 +- .../rest/SliderApplicationApiRestClient.java | 16 +- .../server/appmaster/SliderAppMaster.java | 20 +- .../server/appmaster/management/BoolMetric.java | 88 + .../server/appmaster/management/LongGauge.java | 92 + .../appmaster/management/MeterAndCounter.java | 5 + .../management/MetricsAndMonitoring.java | 24 +- .../management/RangeLimitedCounter.java | 85 + .../server/appmaster/management/Timestamp.java | 33 + .../rpc/SliderClusterProtocolPBImpl.java | 20 + .../rpc/SliderClusterProtocolProxy.java | 20 + .../server/appmaster/rpc/SliderIPCService.java | 78 +- .../slider/server/appmaster/state/AppState.java | 88 +- .../appmaster/state/ProviderAppState.java | 27 +- .../server/appmaster/state/RoleHistory.java | 95 +- .../state/StateAccessForProviders.java | 32 +- .../server/appmaster/web/WebAppApiImpl.java | 3 +- .../server/appmaster/web/rest/RestPaths.java | 3 +- .../ApplicationResouceContentCacheFactory.java | 54 +- .../rest/application/ApplicationResource.java | 36 +- .../resources/LiveNodesRefresher.java | 42 + .../src/main/proto/SliderClusterMessages.proto | 12 + .../src/main/proto/SliderClusterProtocol.proto | 12 +- .../rest/AbstractAppApiTestDelegates.groovy | 17 +- .../agent/rest/IpcApiClientTestDelegates.java | 2 +- .../agent/rest/LowLevelRestTestDelegates.groovy | 6 +- .../slider/agent/rest/TestStandaloneREST.groovy | 77 +- .../TestMockAppStateAppRestIntegration.groovy | 5 +- .../model/monkey/TestMockMonkey.groovy | 2 +- .../publisher/TestPublisherRestResources.groovy | 2 +- .../apache/slider/test/SliderTestUtils.groovy | 53 +- 42 files changed, 3514 insertions(+), 555 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7f8ff5e..3096527 100644 --- a/pom.xml +++ b/pom.xml @@ -575,6 +575,12 @@ <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> <version>${hadoop.version}</version> <exclusions> @@ -590,7 +596,7 @@ <artifactId>hadoop-yarn-registry</artifactId> <version>${hadoop.version}</version> </dependency> - + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-web-proxy</artifactId> @@ -1394,6 +1400,12 @@ <version>1.5</version> </dependency> + <dependency> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + <version>3.3.1</version> + </dependency> + </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/slider-core/pom.xml b/slider-core/pom.xml index 70003c7..7e13b92 100644 --- a/slider-core/pom.xml +++ b/slider-core/pom.xml @@ -260,6 +260,11 @@ <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> <scope>compile</scope> </dependency> @@ -514,7 +519,6 @@ <dependency> <groupId>asm</groupId> <artifactId>asm</artifactId> - <version>3.3.1</version> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java index 7de1915..3668c66 100644 --- a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java +++ b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java @@ -21,6 +21,7 @@ package org.apache.slider.api; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; +import org.apache.slider.api.types.NodeInformation; import org.apache.slider.api.types.PingInformation; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; @@ -101,8 +102,7 @@ public interface SliderApplicationApi { * @return a possibly empty list of serialized containers * @throws IOException on any failure */ - Map<String, ContainerInformation> enumContainers() throws - IOException; + Map<String, ContainerInformation> enumContainers() throws IOException; /** * Get a container from the container Id @@ -110,16 +110,14 @@ public interface SliderApplicationApi { * @return the container information * @throws IOException on any failure */ - ContainerInformation getContainer(String containerId) throws - IOException; + ContainerInformation getContainer(String containerId) throws IOException; /** * List all components into a map of [name:info] * @return a possibly empty map of components * @throws IOException on any failure */ - Map<String, ComponentInformation> enumComponents() throws - IOException; + Map<String, ComponentInformation> enumComponents() throws IOException; /** * Get information about a component @@ -127,8 +125,22 @@ public interface SliderApplicationApi { * @return the component details * @throws IOException on any failure */ - ComponentInformation getComponent(String componentName) throws - IOException; + ComponentInformation getComponent(String componentName) throws IOException; + + /** + * List all nodes into a map of [name:info] + * @return a possibly empty map of nodes + * @throws IOException on any failure + */ + Map<String, NodeInformation> getLiveNodes() throws IOException; + + /** + * Get information about a node + * @param hostname name of the node + * @return the node details + * @throws IOException on any failure + */ + NodeInformation getLiveNode(String hostname) throws IOException; /** * Ping as a GET http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aeab9ca/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 910521e..33fce22 100644 --- a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -30,8 +30,7 @@ import java.io.IOException; * Cluster protocol. This can currently act as a versioned IPC * endpoint or be relayed via protobuf */ -@KerberosInfo( - serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL) +@KerberosInfo(serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL) public interface SliderClusterProtocol extends VersionedProtocol { long versionID = 0x01; @@ -39,10 +38,8 @@ public interface SliderClusterProtocol extends VersionedProtocol { * Stop the cluster */ - Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws IOException, YarnException; - /** * Upgrade the application containers * @@ -155,20 +152,26 @@ public interface SliderClusterProtocol extends VersionedProtocol { Messages.GetLiveComponentRequestProto request ) throws IOException; - + Messages.GetLiveNodesResponseProto getLiveNodes( + Messages.GetLiveNodesRequestProto request + ) throws IOException; + + Messages.NodeInformationProto getLiveNode( + Messages.GetLiveNodeRequestProto request + ) throws IOException; + Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException; - Messages.WrappedJsonProto getModelDesiredAppconf(Messages.EmptyPayloadProto request) throws IOException; - + Messages.WrappedJsonProto getModelDesiredResources(Messages.EmptyPayloadProto request) throws IOException; - + Messages.WrappedJsonProto getModelResolved(Messages.EmptyPayloadProto request) throws IOException; - + Messages.WrappedJsonProto getModelResolvedAppconf(Messages.EmptyPayloadProto request) throws IOException; - + Messages.WrappedJsonProto getModelResolvedResources(Messages.EmptyPayloadProto request) throws IOException; - + Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto request) throws IOException; Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
