YARN-6903. Yarn-native-service framework core rewrite. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/164c0c4c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/164c0c4c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/164c0c4c Branch: refs/heads/yarn-native-services Commit: 164c0c4c9346dd6486f1880c7ea70602f27270b3 Parents: f1a358e Author: Billie Rinaldi <bil...@apache.org> Authored: Mon Aug 14 14:12:12 2017 -0700 Committer: Billie Rinaldi <bil...@apache.org> Committed: Mon Aug 14 14:12:12 2017 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/hadoop-yarn/bin/yarn | 8 +- .../api/impl/ApplicationApiService.java | 139 +-- .../dev-support/findbugs-exclude.xml | 8 + .../hadoop-yarn-slider-core/pom.xml | 1 + .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java | 132 +++ .../yarn/service/ContainerFailureTracker.java | 89 ++ .../yarn/service/ContainerLaunchService.java | 101 +++ .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 132 +++ .../hadoop/yarn/service/ServiceMonitor.java | 143 ++++ .../hadoop/yarn/service/ServiceScheduler.java | 641 ++++++++++++++ .../yarn/service/client/ClientAMProxy.java | 49 ++ .../hadoop/yarn/service/client/ServiceCLI.java | 98 +++ .../yarn/service/client/ServiceClient.java | 836 +++++++++++++++++++ .../client/params/AbstractActionArgs.java | 180 ++++ .../service/client/params/ActionBuildArgs.java | 33 + .../service/client/params/ActionCreateArgs.java | 35 + .../client/params/ActionDependencyArgs.java | 65 ++ .../client/params/ActionDestroyArgs.java | 37 + .../service/client/params/ActionFlexArgs.java | 55 ++ .../yarn/service/client/params/ArgOps.java | 156 ++++ .../yarn/service/client/params/Arguments.java | 138 +++ .../yarn/service/client/params/ClientArgs.java | 383 +++++++++ .../yarn/service/client/params/CommonArgs.java | 294 +++++++ .../client/params/ComponentArgsDelegate.java | 54 ++ .../service/client/params/SliderAMArgs.java | 57 ++ .../client/params/SliderAMCreateAction.java | 77 ++ .../service/client/params/SliderActions.java | 106 +++ .../service/compinstance/ComponentInstance.java | 493 +++++++++++ .../compinstance/ComponentInstanceEvent.java | 58 ++ .../ComponentInstanceEventType.java | 27 + .../compinstance/ComponentInstanceId.java | 91 ++ .../compinstance/ComponentInstanceState.java | 26 + .../yarn/service/component/Component.java | 487 +++++++++++ .../yarn/service/component/ComponentEvent.java | 83 ++ .../service/component/ComponentEventType.java | 26 + .../yarn/service/component/ComponentState.java | 25 + .../yarn/service/conf/SliderExitCodes.java | 88 ++ .../hadoop/yarn/service/conf/SliderKeys.java | 195 +++++ .../yarn/service/conf/SliderXmlConfKeys.java | 191 +++++ .../yarn/service/conf/YarnServiceConfKeys.java | 27 + .../pb/client/ClientAMProtocolPBClientImpl.java | 91 ++ .../impl/pb/service/ClientAMProtocolPB.java | 29 + .../service/ClientAMProtocolPBServiceImpl.java | 70 ++ .../yarn/service/metrics/ServiceMetrics.java | 101 +++ .../provider/AbstractClientProvider.java | 128 +++ .../provider/AbstractProviderService.java | 97 +++ .../yarn/service/provider/ProviderFactory.java | 77 ++ .../yarn/service/provider/ProviderService.java | 37 + .../yarn/service/provider/ProviderUtils.java | 471 +++++++++++ .../defaultImpl/DefaultClientProvider.java | 46 + .../defaultImpl/DefaultProviderFactory.java | 51 ++ .../defaultImpl/DefaultProviderService.java | 36 + .../provider/docker/DockerClientProvider.java | 53 ++ .../service/provider/docker/DockerKeys.java | 30 + .../provider/docker/DockerProviderFactory.java | 52 ++ .../provider/docker/DockerProviderService.java | 57 ++ .../provider/tarball/TarballClientProvider.java | 65 ++ .../tarball/TarballProviderFactory.java | 52 ++ .../tarball/TarballProviderService.java | 48 ++ .../timelineservice/ServiceMetricsSink.java | 102 +++ .../ServiceTimelineEntityType.java | 39 + .../timelineservice/ServiceTimelineEvent.java | 34 + .../ServiceTimelineMetricsConstants.java | 95 +++ .../ServiceTimelinePublisher.java | 428 ++++++++++ .../service/timelineservice/package-info.java | 27 + .../yarn/service/utils/ServiceApiUtil.java | 449 ++++++++++ .../apache/slider/api/ServiceApiConstants.java | 2 +- .../slider/api/SliderClusterProtocol.java | 2 +- .../apache/slider/api/resource/Application.java | 4 +- .../apache/slider/api/resource/Component.java | 3 + .../slider/api/resource/ContainerState.java | 2 +- .../apache/slider/api/resource/Resource.java | 7 + .../org/apache/slider/client/ClientUtils.java | 2 +- .../org/apache/slider/client/SliderClient.java | 167 +++- .../apache/slider/client/SliderClientAPI.java | 4 +- .../slider/client/SliderYarnClientImpl.java | 17 +- .../apache/slider/common/SliderExitCodes.java | 88 -- .../org/apache/slider/common/SliderKeys.java | 195 ----- .../apache/slider/common/SliderXmlConfKeys.java | 190 ----- .../common/params/AbstractActionArgs.java | 179 ---- .../common/params/AbstractArgsDelegate.java | 2 + .../AbstractClusterBuildingActionArgs.java | 2 + .../common/params/ActionAMSuicideArgs.java | 4 +- .../slider/common/params/ActionBuildArgs.java | 31 - .../slider/common/params/ActionClientArgs.java | 4 +- .../slider/common/params/ActionCreateArgs.java | 33 - .../common/params/ActionDependencyArgs.java | 65 -- .../slider/common/params/ActionDestroyArgs.java | 37 - .../common/params/ActionDiagnosticArgs.java | 4 +- .../slider/common/params/ActionExistsArgs.java | 4 +- .../slider/common/params/ActionFlexArgs.java | 55 -- .../slider/common/params/ActionFreezeArgs.java | 4 +- .../slider/common/params/ActionHelpArgs.java | 4 +- .../slider/common/params/ActionKDiagArgs.java | 4 +- .../slider/common/params/ActionKeytabArgs.java | 4 +- .../common/params/ActionKillContainerArgs.java | 4 +- .../slider/common/params/ActionListArgs.java | 4 +- .../slider/common/params/ActionLookupArgs.java | 4 +- .../slider/common/params/ActionNodesArgs.java | 4 +- .../common/params/ActionRegistryArgs.java | 9 +- .../slider/common/params/ActionResolveArgs.java | 6 +- .../common/params/ActionResourceArgs.java | 4 +- .../slider/common/params/ActionStatusArgs.java | 4 +- .../slider/common/params/ActionThawArgs.java | 4 +- .../slider/common/params/ActionTokensArgs.java | 4 +- .../slider/common/params/ActionUpdateArgs.java | 3 +- .../slider/common/params/ActionUpgradeArgs.java | 5 +- .../slider/common/params/ActionVersionArgs.java | 4 +- .../org/apache/slider/common/params/ArgOps.java | 157 ---- .../apache/slider/common/params/Arguments.java | 138 --- .../apache/slider/common/params/ClientArgs.java | 361 -------- .../apache/slider/common/params/CommonArgs.java | 300 ------- .../common/params/ComponentArgsDelegate.java | 52 -- .../slider/common/params/SliderAMArgs.java | 57 -- .../common/params/SliderAMCreateAction.java | 74 -- .../slider/common/params/SliderActions.java | 106 --- .../slider/common/tools/ConfigHelper.java | 4 +- .../slider/common/tools/CoreFileSystem.java | 12 +- .../apache/slider/common/tools/PortScanner.java | 2 +- .../apache/slider/common/tools/SliderUtils.java | 18 +- .../slider/core/exceptions/SliderException.java | 2 +- .../slider/core/launch/AbstractLauncher.java | 253 +----- .../slider/core/launch/ContainerLauncher.java | 26 +- .../slider/core/launch/CredentialUtils.java | 2 +- .../core/registry/SliderRegistryUtils.java | 2 +- .../providers/AbstractClientProvider.java | 153 ---- .../providers/AbstractProviderService.java | 36 +- .../slider/providers/DefaultClientProvider.java | 45 - .../providers/DefaultProviderFactory.java | 47 -- .../providers/DefaultProviderService.java | 38 - .../slider/providers/ProviderService.java | 68 -- .../apache/slider/providers/ProviderUtils.java | 525 ------------ .../slider/providers/SliderProviderFactory.java | 77 -- .../providers/docker/DockerClientProvider.java | 53 -- .../slider/providers/docker/DockerKeys.java | 30 - .../providers/docker/DockerProviderFactory.java | 52 -- .../providers/docker/DockerProviderService.java | 66 -- .../tarball/TarballClientProvider.java | 65 -- .../tarball/TarballProviderFactory.java | 52 -- .../tarball/TarballProviderService.java | 54 -- .../server/appmaster/RoleLaunchService.java | 105 +-- .../server/appmaster/SliderAppMaster.java | 126 ++- .../actions/ProviderReportedContainerLoss.java | 53 -- .../server/appmaster/metrics/SliderMetrics.java | 107 --- .../appmaster/monkey/ChaosKillContainer.java | 2 +- .../slider/server/appmaster/rpc/RpcBinder.java | 5 +- .../appmaster/rpc/SliderAMPolicyProvider.java | 2 +- .../appmaster/rpc/SliderRPCSecurityInfo.java | 2 +- .../security/SecurityConfiguration.java | 4 +- .../slider/server/appmaster/state/AppState.java | 62 +- .../appmaster/state/AppStateBindingInfo.java | 3 - .../server/appmaster/state/RoleHistory.java | 1 - .../server/appmaster/state/RoleInstance.java | 1 - .../server/appmaster/state/RoleStatus.java | 19 +- .../ServiceTimelinePublisher.java | 372 --------- .../timelineservice/SliderMetricsSink.java | 102 --- .../SliderTimelineEntityType.java | 39 - .../timelineservice/SliderTimelineEvent.java | 34 - .../SliderTimelineMetricsConstants.java | 93 --- .../appmaster/timelineservice/package-info.java | 27 - .../server/appmaster/web/view/IndexBlock.java | 4 +- .../slider/server/avro/RoleHistoryWriter.java | 2 +- .../slider/server/servicemonitor/HttpProbe.java | 20 +- .../slider/server/servicemonitor/PortProbe.java | 22 +- .../slider/server/servicemonitor/Probe.java | 6 +- .../YarnRegistryViewForProviders.java | 64 +- .../apache/slider/util/RestApiConstants.java | 4 +- .../org/apache/slider/util/ServiceApiUtil.java | 407 --------- .../src/main/proto/ClientAMProtocol.proto | 56 ++ .../hadoop/yarn/service/TestServiceApiUtil.java | 539 ++++++++++++ .../yarn/service/TestYarnNativeServices.java | 496 +++++++++++ .../client/TestBuildExternalComponents.java | 128 +++ .../yarn/service/client/TestServiceCLI.java | 134 +++ .../yarn/service/conf/ExampleAppJson.java | 64 ++ .../yarn/service/conf/TestAppJsonResolve.java | 252 ++++++ .../service/conf/TestLoadExampleAppJson.java | 81 ++ .../service/conf/TestValidateServiceNames.java | 123 +++ .../providers/TestAbstractClientProvider.java | 119 +++ .../service/providers/TestProviderFactory.java | 75 ++ .../TestServiceTimelinePublisher.java | 311 +++++++ .../org/apache/slider/api/TestRPCBinding.java | 4 +- .../apache/slider/client/TestClientBadArgs.java | 34 +- .../slider/client/TestClientBasicArgs.java | 10 +- .../slider/client/TestCommonArgParsing.java | 80 +- .../slider/client/TestKeytabCommandOptions.java | 22 +- .../slider/client/TestSliderClientMethods.java | 8 +- .../slider/client/TestSliderTokensCommand.java | 18 +- .../slider/common/tools/TestClusterNames.java | 122 --- .../slider/common/tools/TestConfigHelper.java | 4 +- .../common/tools/TestConfigHelperHDFS.java | 2 +- .../common/tools/TestExecutionEnvironment.java | 14 +- .../slider/common/tools/TestPortScan.java | 26 +- .../common/tools/TestSliderFileSystem.java | 6 +- .../common/tools/TestSliderTestUtils.java | 12 +- .../slider/common/tools/TestSliderUtils.java | 10 +- .../slider/common/tools/TestWindowsSupport.java | 24 +- .../slider/common/tools/TestZKIntegration.java | 8 +- .../apache/slider/core/conf/ExampleAppJson.java | 64 -- .../core/conf/TestConfigurationResolve.java | 252 ------ .../slider/core/conf/TestExampleAppJson.java | 81 -- .../TestPublishedConfigurationOutputter.java | 14 +- .../slider/other/TestFilesystemPermissions.java | 12 +- .../apache/slider/other/TestLocalDirStatus.java | 8 +- .../providers/TestAbstractClientProvider.java | 3 +- .../TestBuildApplicationComponent.java | 96 --- .../slider/providers/TestDefaultProvider.java | 60 -- .../slider/providers/TestProviderFactory.java | 37 +- .../slider/registry/TestConfigSetNaming.java | 10 +- .../slider/registry/TestRegistryPaths.java | 8 +- .../server/appmaster/actions/TestActions.java | 16 +- .../TestMockAppStateAAOvercapacity.java | 2 +- .../appstate/TestMockAppStateAAPlacement.java | 16 +- .../TestMockAppStateContainerFailure.java | 30 +- .../appstate/TestMockAppStateDependencies.java | 7 +- .../TestMockAppStateDynamicHistory.java | 4 +- .../appstate/TestMockAppStateDynamicRoles.java | 12 +- .../TestMockAppStateFlexDynamicRoles.java | 6 +- .../model/appstate/TestMockAppStateFlexing.java | 10 +- .../appstate/TestMockAppStateRMOperations.java | 28 +- .../TestMockAppStateRebuildOnAMRestart.java | 2 +- .../appstate/TestMockAppStateRolePlacement.java | 2 +- .../appstate/TestMockAppStateRoleRelease.java | 2 +- .../appstate/TestMockAppStateUniqueNames.java | 4 +- .../TestMockContainerResourceAllocations.java | 6 +- .../appstate/TestMockLabelledAAPlacement.java | 4 +- .../TestOutstandingRequestValidation.java | 8 +- .../model/history/TestRoleHistoryAA.java | 26 +- .../history/TestRoleHistoryContainerEvents.java | 18 +- ...TestRoleHistoryFindNodesForNewInstances.java | 14 +- .../history/TestRoleHistoryNIComparators.java | 12 +- ...estRoleHistoryOutstandingRequestTracker.java | 26 +- .../model/history/TestRoleHistoryRW.java | 22 +- .../history/TestRoleHistoryRWOrdering.java | 12 +- .../history/TestRoleHistoryRequestTracking.java | 20 +- .../history/TestRoleHistoryUpdateBlacklist.java | 4 +- .../model/mock/BaseMockAppStateTest.java | 2 +- .../model/mock/MockProviderService.java | 149 ---- .../appmaster/model/monkey/TestMockMonkey.java | 20 +- .../security/TestSecurityConfiguration.java | 22 +- .../TestServiceTimelinePublisher.java | 304 ------- .../appmaster/timelineservice/package-info.java | 26 - .../web/rest/registry/PathEntryMarshalling.java | 2 +- .../registry/TestRegistryRestMarshalling.java | 2 +- .../web/view/TestClusterSpecificationBlock.java | 2 +- .../web/view/TestContainerStatsBlock.java | 14 +- .../appmaster/web/view/TestIndexBlock.java | 2 +- .../slider/server/management/TestGauges.java | 2 +- .../server/servicemonitor/TestPortProbe.java | 42 - .../workflow/TestWorkflowClosingService.java | 10 +- .../workflow/TestWorkflowCompositeService.java | 14 +- .../workflow/TestWorkflowExecutorService.java | 4 +- .../workflow/TestWorkflowRpcService.java | 6 +- .../workflow/TestWorkflowSequenceService.java | 18 +- .../TestWorkflowServiceTerminatingRunnable.java | 6 +- .../org/apache/slider/utils/KeysForTests.java | 4 +- .../apache/slider/utils/SliderTestUtils.java | 4 +- .../org/apache/slider/utils/TestAssertions.java | 6 +- .../apache/slider/utils/TestServiceApiUtil.java | 523 ------------ .../slider/utils/YarnMiniClusterTestBase.java | 8 +- .../src/test/resources/example-app.json | 15 + .../service/conf/examples/app-override.json | 72 ++ .../hadoop/yarn/service/conf/examples/app.json | 47 ++ .../yarn/service/conf/examples/default.json | 16 + .../yarn/service/conf/examples/external0.json | 8 + .../yarn/service/conf/examples/external1.json | 30 + .../yarn/service/conf/examples/external2.json | 22 + .../yarn/service/provider/docker/appConfig.json | 42 + .../yarn/service/provider/docker/resources.json | 16 + .../yarn/service/provider/docker/test.template | 16 + .../slider/core/conf/examples/app-override.json | 72 -- .../apache/slider/core/conf/examples/app.json | 47 -- .../slider/core/conf/examples/default.json | 16 - .../slider/core/conf/examples/external0.json | 8 - .../slider/core/conf/examples/external1.json | 30 - .../slider/core/conf/examples/external2.json | 22 - .../slider/providers/docker/appConfig.json | 42 - .../slider/providers/docker/resources.json | 16 - .../slider/providers/docker/test.template | 16 - .../src/test/resources/yarn-site.xml | 19 + .../hadoop/yarn/client/api/AMRMClient.java | 3 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 34 + .../hadoop/yarn/util/BoundedAppender.java | 133 +++ .../hadoop/yarn/util/TestBoundedAppender.java | 115 +++ .../client/api/RegistryOperationsFactory.java | 8 + .../registry/client/impl/zk/CuratorService.java | 5 +- .../impl/zk/RegistryOperationsService.java | 4 +- .../server/resourcemanager/RMAppManager.java | 56 +- .../blacklist/SimpleBlacklistManager.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 147 +--- .../scheduler/AbstractYarnScheduler.java | 11 +- .../scheduler/SchedulerUtils.java | 17 +- .../server/resourcemanager/TestRMRestart.java | 8 +- .../rmapp/attempt/TestBoundedAppender.java | 116 --- .../TestRMAppAttemptImplDiagnostics.java | 3 +- 297 files changed, 12492 insertions(+), 8197 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 39ef8d2..98a2036 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -47,9 +47,9 @@ function hadoop_usage hadoop_add_subcommand "resourcemanager" "run the ResourceManager" hadoop_add_subcommand "rmadmin" "admin tools" hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools" - hadoop_add_subcommand "servicesapi" "run slider services api" + hadoop_add_subcommand "servicesapi" "run yarn-service rest server" hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon" - hadoop_add_subcommand "slider" "run a slider app" + hadoop_add_subcommand "service" "run a service" hadoop_add_subcommand "timelinereader" "run the timeline reader server" hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "top" "view cluster information" @@ -161,9 +161,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; - slider) + service) hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' - HADOOP_CLASSNAME='org.apache.slider.Slider' + HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 5a4de0e..decd849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -18,23 +18,21 @@ package org.apache.hadoop.yarn.services.api.impl; import com.google.inject.Singleton; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.ApplicationState; import org.apache.slider.api.resource.ApplicationStatus; import org.apache.slider.api.resource.Component; -import org.apache.slider.util.ServiceApiUtil; -import org.apache.slider.client.SliderClient; -import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.BuildHelper; -import org.apache.slider.core.exceptions.SliderException; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +49,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import static org.apache.slider.util.RestApiConstants.*; @@ -61,51 +58,39 @@ import static org.apache.slider.util.RestApiConstants.*; @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public class ApplicationApiService { - private static final Logger logger = + private static final Logger LOG = LoggerFactory.getLogger(ApplicationApiService.class); - private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG = - new YarnConfiguration(); - private static SliderClient SLIDER_CLIENT; - private static Response SLIDER_VERSION; - private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs(); + private static Configuration YARN_CONFIG = new YarnConfiguration(); + private static ServiceClient SERVICE_CLIENT; static { init(); } // initialize all the common resources - order is important - protected static void init() { - SLIDER_CLIENT = createSliderClient(); - SLIDER_VERSION = initSliderVersion(); + private static void init() { + SERVICE_CLIENT = new ServiceClient(); + SERVICE_CLIENT.init(YARN_CONFIG); + SERVICE_CLIENT.start(); } @GET - @Path("/versions/slider-version") + @Path("/versions/yarn-service-version") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() { - logger.info("GET: getSliderVersion"); - return SLIDER_VERSION; - } - - private static Response initSliderVersion() { - Map<String, Object> metadata = new HashMap<>(); - BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services"); - String sliderVersion = metadata.toString(); - logger.info("Slider version = {}", sliderVersion); - String hadoopVersion = SliderVersionInfo.getHadoopVersionString(); - logger.info("Hadoop version = {}", hadoopVersion); - return Response.ok("{ \"slider_version\": \"" + sliderVersion - + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); + String version = VersionInfo.getBuildVersion(); + LOG.info(version); + return Response.ok(version).build(); } @POST @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response createApplication(Application application) { - logger.info("POST: createApplication = {}", application); + LOG.info("POST: createApplication = {}", application); ApplicationStatus applicationStatus = new ApplicationStatus(); try { - ApplicationId applicationId = SLIDER_CLIENT.actionCreate(application); - logger.info("Successfully created application " + application.getName() + ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application); + LOG.info("Successfully created application " + application.getName() + " applicationId = " + applicationId); applicationStatus.setState(ApplicationState.ACCEPTED); applicationStatus.setUri( @@ -118,58 +103,18 @@ public class ApplicationApiService { .build(); } catch (Exception e) { String message = "Failed to create application " + application.getName(); - logger.error(message, e); + LOG.error(message, e); applicationStatus.setDiagnostics(message + ": " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - protected static SliderClient createSliderClient() { - if (SLIDER_CLIENT != null) { - return SLIDER_CLIENT; - } - org.apache.hadoop.conf.Configuration sliderClientConfiguration = - SLIDER_CONFIG; - SliderClient client = new SliderClient() { - @Override public void init(org.apache.hadoop.conf.Configuration conf) { - super.init(conf); - try { - initHadoopBinding(); - } catch (SliderException | IOException e) { - throw new RuntimeException( - "Unable to automatically init Hadoop binding", e); - } - } - }; - try { - logger - .debug("Slider Client configuration: {}", sliderClientConfiguration); - sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "help" }); - client.init(sliderClientConfiguration); - client.start(); - } catch (Exception e) { - logger.error("Unable to create SliderClient", e); - throw new RuntimeException(e.getMessage(), e); - } - return client; - } - - // The information this REST endpoint currently returned can be retrieved from - // RM web services - // Probably the data from AM is more important. Do that later. -// @GET @Consumes({ MediaType.APPLICATION_JSON }) -// @Produces({ MediaType.APPLICATION_JSON }) -// public Response getApplications(@QueryParam("state") String state) { -// logger.info("GET: getApplications with param state = {}", state); -// return null; -// } - @GET @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getApplication(@PathParam("app_name") String appName) { - logger.info("GET: getApplication for appName = {}", appName); + LOG.info("GET: getApplication for appName = {}", appName); ApplicationStatus applicationStatus = new ApplicationStatus(); // app name validation @@ -181,24 +126,25 @@ public class ApplicationApiService { } try { - Application app = SLIDER_CLIENT.actionStatus(appName); - ApplicationReport report = SLIDER_CLIENT.findInstance(appName); - if (app != null && report != null) { + Application app = SERVICE_CLIENT.getStatus(appName); + ApplicationReport report = SERVICE_CLIENT.getYarnClient() + .getApplicationReport(ApplicationId.fromString(app.getId())); + if (report != null) { app.setLifetime( report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME) .getRemainingTime()); - logger.info("Application = {}", app); + LOG.info("Application = {}", app); return Response.ok(app).build(); } else { String message = "Application " + appName + " does not exist."; - logger.info(message); + LOG.info(message); applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); applicationStatus.setDiagnostics(message); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } } catch (Exception e) { - logger.error("Get application failed", e); + LOG.error("Get application failed", e); applicationStatus .setDiagnostics("Failed to retrieve application: " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) @@ -211,18 +157,18 @@ public class ApplicationApiService { @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response deleteApplication(@PathParam("app_name") String appName) { - logger.info("DELETE: deleteApplication for appName = {}", appName); + LOG.info("DELETE: deleteApplication for appName = {}", appName); return stopApplication(appName, true); } private Response stopApplication(String appName, boolean destroy) { try { - SLIDER_CLIENT.actionStop(appName, ACTION_FREEZE_ARGS); + SERVICE_CLIENT.actionStop(appName); if (destroy) { - SLIDER_CLIENT.actionDestroy(appName); - logger.info("Successfully deleted application {}", appName); + SERVICE_CLIENT.actionDestroy(appName); + LOG.info("Successfully deleted application {}", appName); } else { - logger.info("Successfully stopped application {}", appName); + LOG.info("Successfully stopped application {}", appName); } return Response.status(Status.NO_CONTENT).build(); } catch (ApplicationNotFoundException e) { @@ -252,8 +198,8 @@ public class ApplicationApiService { .getNumberOfContainers()).build(); } try { - Map<String, Long> original = SLIDER_CLIENT.flex(appName, Collections - .singletonMap(component.getName(), + Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName, + Collections.singletonMap(component.getName(), component.getNumberOfContainers())); return Response.ok().entity("Updating " + componentName + " size from " + original.get(componentName) + " to " @@ -271,7 +217,7 @@ public class ApplicationApiService { @Produces({ MediaType.APPLICATION_JSON }) public Response updateApplication(@PathParam("app_name") String appName, Application updateAppData) { - logger.info("PUT: updateApplication for app = {} with data = {}", appName, + LOG.info("PUT: updateApplication for app = {} with data = {}", appName, updateAppData); // Ignore the app name provided in updateAppData and always use appName @@ -314,14 +260,14 @@ public class ApplicationApiService { private Response updateLifetime(String appName, Application updateAppData) { try { String newLifeTime = - SLIDER_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); return Response.ok("Application " + appName + " lifeTime is successfully updated to " + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); } catch (Exception e) { String message = "Failed to update application (" + appName + ") lifetime (" + updateAppData.getLifetime() + ")"; - logger.error(message, e); + LOG.error(message, e); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(message + " : " + e.getMessage()).build(); } @@ -329,17 +275,12 @@ public class ApplicationApiService { private Response startApplication(String appName) { try { - int ret = SLIDER_CLIENT.actionList(appName); - if (ret == 0) { - return Response.ok() - .entity("Application " + appName + " is already alive.").build(); - } - SLIDER_CLIENT.actionStart(appName, null); - logger.info("Successfully started application " + appName); + SERVICE_CLIENT.actionStart(appName); + LOG.info("Successfully started application " + appName); return Response.ok("Application " + appName + " is successfully started").build(); } catch (Exception e) { String message = "Failed to start application " + appName; - logger.info(message, e); + LOG.info(message, e); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(message + ": " + e.getMessage()).build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml index e5cde4e..f2bf582 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml @@ -17,6 +17,14 @@ --> <FindBugsFilter> <Match> + <Package name="org.apache.hadoop.yarn.proto" /> + </Match> + <Match> + <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" /> + <Bug pattern="MS_CANNOT_BE_FINAL" /> + </Match> + + <Match> <Package name="org.apache.slider.api.proto" /> </Match> <Match> http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index a1d1937..02317e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -61,6 +61,7 @@ <includes> <include>SliderClusterMessages.proto</include> <include>SliderClusterProtocol.proto</include> + <include>ClientAMProtocol.proto</include> </includes> </source> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java new file mode 100644 index 0000000..516d23d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; + +import java.io.IOException; + +public interface ClientAMProtocol { + FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request) + throws IOException, YarnException; + + GetStatusResponseProto getStatus(GetStatusRequestProto requestProto) + throws IOException, YarnException; + + StopResponseProto stop(StopRequestProto requestProto) + throws IOException, YarnException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java new file mode 100644 index 0000000..6884757 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto; +import org.apache.hadoop.yarn.service.component.ComponentEvent; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX; + +public class ClientAMService extends AbstractService + implements ClientAMProtocol { + + private static final Logger LOG = + LoggerFactory.getLogger(ClientAMService.class); + + private ServiceContext context; + private Server server; + + private InetSocketAddress bindAddress; + + public ClientAMService(ServiceContext context) { + super("Client AM Service"); + this.context = context; + } + + @Override protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress address = new InetSocketAddress(0); + server = rpc.getServer(ClientAMProtocol.class, this, address, conf, + context.secretManager, 1); + server.start(); + + String nodeHostString = + System.getenv(ApplicationConstants.Environment.NM_HOST.name()); + + bindAddress = NetUtils.createSocketAddrForHost(nodeHostString, + server.getListenerAddress().getPort()); + + LOG.info("Instantiated ClientAMService at " + bindAddress); + super.serviceStart(); + } + + @Override protected void serviceStop() throws Exception { + if (server != null) { + server.stop(); + } + super.serviceStop(); + } + + @Override public FlexComponentsResponseProto flexComponents( + FlexComponentsRequestProto request) throws IOException { + if (!request.getComponentsList().isEmpty()) { + for (ComponentCountProto component : request.getComponentsList()) { + ComponentEvent event = new ComponentEvent(component.getName(), FLEX) + .setDesired(component.getNumberOfContainers()); + context.scheduler.getDispatcher().getEventHandler().handle(event); + LOG.info("Flexing component {} to {}", component.getName(), + component.getNumberOfContainers()); + } + } + return FlexComponentsResponseProto.newBuilder().build(); + } + + @Override + public GetStatusResponseProto getStatus(GetStatusRequestProto request) + throws IOException, YarnException { + String stat = ServiceApiUtil.jsonSerDeser.toJson(context.application); + return GetStatusResponseProto.newBuilder().setStatus(stat).build(); + } + + @Override + public StopResponseProto stop(StopRequestProto requestProto) + throws IOException, YarnException { + LOG.info("Stop the service."); + // Stop the service in 2 seconds delay to make sure this rpc call is completed. + // shutdown hook will be executed which will stop AM gracefully. + Thread thread = new Thread() { + @Override + public void run() { + try { + Thread.sleep(2000); + ExitUtil.terminate(0); + } catch (InterruptedException e) { + LOG.error("Interrupted while stopping", e); + } + } + }; + thread.start(); + return StopResponseProto.newBuilder().build(); + } + + public InetSocketAddress getBindAddress() { + return bindAddress; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java new file mode 100644 index 0000000..bbb4c44 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.yarn.service.component.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD; + +/** + * This tracks the container failures per node. If the failure counter exceeds + * the maxFailurePerNode limit, it'll blacklist that node. + * + */ +public class ContainerFailureTracker { + + private static final Logger LOG = + LoggerFactory.getLogger(ContainerFailureTracker.class); + + // Host -> num container failures + private Map<String, Integer> failureCountPerNode = new HashMap<>(); + private Set<String> blackListedNodes = new HashSet<>(); + private ServiceContext context; + private int maxFailurePerNode; + private Component component; + + public ContainerFailureTracker(ServiceContext context, Component component) { + this.context = context; + this.component = component; + maxFailurePerNode = component.getComponentSpec().getConfiguration() + .getPropertyInt(NODE_FAILURE_THRESHOLD, 3); + } + + + public synchronized void incNodeFailure(String host) { + int num = 0; + if (failureCountPerNode.containsKey(host)) { + num = failureCountPerNode.get(host); + } + num++; + failureCountPerNode.put(host, num); + + // black list the node if exceed max failure + if (num > maxFailurePerNode && !blackListedNodes.contains(host)) { + List<String> blacklists = new ArrayList<>(); + blacklists.add(host); + blackListedNodes.add(host); + context.scheduler.getAmRMClient().updateBlacklist(blacklists, null); + LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}." + + " Current list of blacklisted nodes: {}", + component.getName(), num, host, blackListedNodes); + } + } + + public synchronized void resetContainerFailures() { + // reset container failure counter per node + failureCountPerNode.clear(); + context.scheduler.getAmRMClient() + .updateBlacklist(null, new ArrayList<>(blackListedNodes)); + LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ", + component.getName(), blackListedNodes); + blackListedNodes.clear(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java new file mode 100644 index 0000000..ac5285f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; +import org.apache.hadoop.yarn.service.provider.ProviderService; +import org.apache.hadoop.yarn.service.provider.ProviderFactory; +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.launch.AbstractLauncher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ContainerLaunchService extends AbstractService{ + + protected static final Logger LOG = + LoggerFactory.getLogger(ContainerLaunchService.class); + + private ExecutorService executorService; + private SliderFileSystem fs; + + public ContainerLaunchService(SliderFileSystem fs) { + super(ContainerLaunchService.class.getName()); + this.fs = fs; + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + executorService = Executors.newCachedThreadPool(); + super.serviceInit(conf); + } + + @Override + protected void serviceStop() throws Exception { + if (executorService != null) { + executorService.shutdownNow(); + } + super.serviceStop(); + } + + public void launchCompInstance(Application application, + ComponentInstance instance, Container container) { + ContainerLauncher launcher = + new ContainerLauncher(application, instance, container); + executorService.execute(launcher); + } + + private class ContainerLauncher implements Runnable { + public final Container container; + public final Application application; + public ComponentInstance instance; + + public ContainerLauncher( + Application application, + ComponentInstance instance, Container container) { + this.container = container; + this.application = application; + this.instance = instance; + } + + @Override public void run() { + org.apache.slider.api.resource.Component compSpec = instance.getCompSpec(); + ProviderService provider = ProviderFactory.getProviderService( + compSpec.getArtifact()); + AbstractLauncher launcher = new AbstractLauncher(fs, null); + try { + provider.buildContainerLaunchContext(launcher, application, + instance, fs); + instance.getComponent().getScheduler().getNmClient() + .startContainerAsync(container, + launcher.completeContainerLaunch()); + } catch (Exception e) { + LOG.error(instance.getCompInstanceId() + + ": Failed to launch container. ", e); + + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java new file mode 100644 index 0000000..80668a0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import com.google.common.cache.LoadingCache; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.common.tools.SliderFileSystem; + +public class ServiceContext { + public Application application = null; + public SliderFileSystem fs; + public String serviceHdfsDir = ""; + public ApplicationAttemptId attemptId; + public LoadingCache<ConfigFile, Object> configCache; + public ServiceScheduler scheduler; + public ClientToAMTokenSecretManager secretManager; + public ClientAMService clientAMService; + + public ServiceContext() { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java new file mode 100644 index 0000000..1ebd562 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; +import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class ServiceMaster extends CompositeService { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceMaster.class); + + private static SliderAMArgs amArgs; + + public ServiceMaster(String name) { + super(name); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + //TODO Deprecate slider conf, make sure works with yarn conf + printSystemEnv(); + if (UserGroupInformation.isSecurityEnabled()) { + UserGroupInformation.setConfiguration(conf); + } + LOG.info("Login user is {}", UserGroupInformation.getLoginUser()); + + ServiceContext context = new ServiceContext(); + Path appDir = new Path(amArgs.getAppDefPath()).getParent(); + SliderFileSystem fs = new SliderFileSystem(conf); + context.fs = fs; + fs.setAppDir(appDir); + context.application = ServiceApiUtil + .loadApplicationFrom(fs, new Path(amArgs.getAppDefPath())); + LOG.info(context.application.toString()); + ContainerId amContainerId = ContainerId.fromString(SliderUtils + .mandatoryEnvVariable( + ApplicationConstants.Environment.CONTAINER_ID.name())); + ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId(); + LOG.info("Application attemptId: " + attemptId); + context.attemptId = attemptId; + + // configure AM to wait forever for RM + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1); + conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS); + + DefaultMetricsSystem.initialize("ServiceAppMaster"); + + context.secretManager = new ClientToAMTokenSecretManager(attemptId, null); + ClientAMService clientAMService = new ClientAMService(context); + context.clientAMService = clientAMService; + addService(clientAMService); + + ServiceScheduler scheduler = new ServiceScheduler(context); + addService(scheduler); + context.scheduler = scheduler; + + ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context); + addService(monitor); + + super.serviceInit(conf); + } + + + @Override + protected void serviceStop() throws Exception { + LOG.info("Stopping app master"); + super.serviceStop(); + } + + private void printSystemEnv() { + for (Map.Entry<String, String> envs : System.getenv().entrySet()) { + LOG.info("{} = {}", envs.getKey(), envs.getValue()); + } + } + + public static void main(String[] args) throws Exception { + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG); + amArgs = new SliderAMArgs(args); + amArgs.parse(); + try { + ServiceMaster serviceMaster = new ServiceMaster("Service Master"); + ShutdownHookManager.get() + .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30); + YarnConfiguration conf = new YarnConfiguration(); + new GenericOptionsParser(conf, args); + serviceMaster.init(conf); + serviceMaster.start(); + } catch (Throwable t) { + LOG.error("Error starting service master", t); + ExitUtil.terminate(1, "Error starting service master"); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java new file mode 100644 index 0000000..82d768e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.service.component.Component; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; +import org.apache.slider.api.InternalKeys; +import org.apache.hadoop.yarn.service.component.ComponentEvent; +import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.component.ComponentState; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.server.servicemonitor.ProbeStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX; +import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY; +import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY; +import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY; + +public class ServiceMonitor extends AbstractService { + + private static final Logger LOG = + LoggerFactory.getLogger(ServiceMonitor.class); + + public ScheduledExecutorService executorService; + private Map<ContainerId, ComponentInstance> liveInstances = null; + private ServiceContext context; + + public ServiceMonitor(String name, ServiceContext context) { + super(name); + liveInstances = context.scheduler.getLiveInstances(); + this.context = context; + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + executorService = Executors.newScheduledThreadPool(1); + super.serviceInit(conf); + } + + @Override + public void serviceStart() throws Exception { + long readinessCheckInterval = context.application.getConfiguration() + .getPropertyLong(InternalKeys.MONITOR_INTERVAL, + InternalKeys.DEFAULT_MONITOR_INTERVAL); + executorService + .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval, + readinessCheckInterval, TimeUnit.SECONDS); + + long failureResetInterval = SliderUtils + .getTimeRange(context.application.getConfiguration(), + ResourceKeys.CONTAINER_FAILURE_WINDOW, + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS, + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS, + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0); + + executorService + .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval, + failureResetInterval, TimeUnit.SECONDS); + } + + @Override + public void serviceStop() throws Exception { + if (executorService != null) { + executorService.shutdownNow(); + } + } + + private class ReadinessChecker implements Runnable { + + @Override + public void run() { + + // check if the comp instance are ready + for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances + .entrySet()) { + ComponentInstance instance = entry.getValue(); + + ProbeStatus status = instance.ping(); + if (status.isSuccess()) { + if (instance.getState() != READY) { + // synchronously update the state. + instance.handle( + new ComponentInstanceEvent(entry.getKey(), BECOME_READY)); + } + } else { + if (instance.getState() == READY) { + instance.handle( + new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY)); + } + } + } + + for (Component component : context.scheduler.getAllComponents() + .values()) { + // If comp hasn't started yet and its dependencies are satisfied + if (component.getState() == ComponentState.INIT && component + .areDependenciesReady()) { + LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.", + component.getName()); + ComponentEvent event = new ComponentEvent(component.getName(), FLEX) + .setDesired(component.getComponentSpec().getNumberOfContainers()); + component.handle(event); + } + } + } + } + + private class ContainerFailureReset implements Runnable { + @Override + public void run() { + for (Component component : context.scheduler.getAllComponents().values()) { + component.resetCompFailureCount(); + } + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org