Merge branch 'yarn-native-services' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbc632d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbc632d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbc632d9 Branch: refs/heads/YARN-5881 Commit: cbc632d9abf08c56a7fc02be51b2718af30bad28 Parents: dcd99c4 a55d073 Author: Jian He <[email protected]> Authored: Mon Nov 6 14:02:19 2017 -0800 Committer: Jian He <[email protected]> Committed: Mon Nov 6 14:02:19 2017 -0800 ---------------------------------------------------------------------- LICENSE.txt | 1 + NOTICE.txt | 10 + .../resources/assemblies/hadoop-yarn-dist.xml | 25 + hadoop-project/pom.xml | 44 +- hadoop-project/src/site/site.xml | 10 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 18 +- .../hadoop-yarn/conf/yarn-env.sh | 18 + .../dev-support/findbugs-exclude.xml | 16 +- .../hadoop/yarn/conf/YarnConfiguration.java | 2 + .../dev-support/findbugs-exclude.xml | 20 + .../hadoop-yarn-services-api/pom.xml | 130 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 298 +++ .../yarn/service/webapp/ApiServerWebApp.java | 161 ++ .../definition/YARN-Services-Examples.md | 245 +++ ...RN-Simplified-V1-API-Layer-For-Services.yaml | 471 +++++ .../src/main/resources/log4j-server.properties | 76 + .../src/main/resources/webapps/api-server/app | 16 + .../src/main/webapp/WEB-INF/web.xml | 36 + .../hadoop/yarn/service/ServiceClientTest.java | 107 ++ .../hadoop/yarn/service/TestApiServer.java | 366 ++++ .../dev-support/findbugs-exclude.xml | 48 + .../conf/yarnservice-log4j.properties | 62 + .../examples/httpd-no-dns/httpd-no-dns.json | 62 + .../httpd-no-dns/httpd-proxy-no-dns.conf | 24 + .../examples/httpd/httpd-proxy.conf | 24 + .../examples/httpd/httpd.json | 55 + .../examples/sleeper/sleeper.json | 15 + .../hadoop-yarn-services-core/pom.xml | 255 +++ .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java | 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 169 ++ .../hadoop/yarn/service/ServiceMetrics.java | 94 + .../hadoop/yarn/service/ServiceScheduler.java | 691 +++++++ .../yarn/service/api/ServiceApiConstants.java | 74 + .../yarn/service/api/records/Artifact.java | 168 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 430 +++++ .../service/api/records/ComponentState.java | 30 + .../yarn/service/api/records/ConfigFile.java | 233 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 298 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java | 102 + .../service/api/records/ReadinessCheck.java | 183 ++ .../yarn/service/api/records/Resource.java | 161 ++ .../yarn/service/api/records/Service.java | 390 ++++ .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../yarn/service/client/ServiceClient.java | 960 ++++++++++ .../yarn/service/component/Component.java | 584 ++++++ .../yarn/service/component/ComponentEvent.java | 83 + .../service/component/ComponentEventType.java | 27 + .../yarn/service/component/ComponentState.java | 25 + .../component/instance/ComponentInstance.java | 549 ++++++ .../instance/ComponentInstanceEvent.java | 58 + .../instance/ComponentInstanceEventType.java | 26 + .../component/instance/ComponentInstanceId.java | 91 + .../instance/ComponentInstanceState.java | 26 + .../yarn/service/conf/RestApiConstants.java | 39 + .../yarn/service/conf/SliderExitCodes.java | 88 + .../yarn/service/conf/YarnServiceConf.java | 113 ++ .../yarn/service/conf/YarnServiceConstants.java | 92 + .../containerlaunch/AbstractLauncher.java | 271 +++ .../containerlaunch/ClasspathConstructor.java | 172 ++ .../containerlaunch/CommandLineBuilder.java | 86 + .../containerlaunch/ContainerLaunchService.java | 101 + .../containerlaunch/CredentialUtils.java | 319 ++++ .../containerlaunch/JavaCommandLineBuilder.java | 181 ++ .../exceptions/BadClusterStateException.java | 36 + .../BadCommandArgumentsException.java | 30 + .../service/exceptions/BadConfigException.java | 39 + .../yarn/service/exceptions/ErrorStrings.java | 42 + .../service/exceptions/ExitCodeProvider.java | 32 + .../service/exceptions/LauncherExitCodes.java | 196 ++ .../exceptions/RestApiErrorMessages.java | 92 + .../exceptions/ServiceLaunchException.java | 73 + .../service/exceptions/SliderException.java | 66 + .../yarn/service/exceptions/UsageException.java | 34 + .../pb/client/ClientAMProtocolPBClientImpl.java | 91 + .../impl/pb/service/ClientAMProtocolPB.java | 29 + .../service/ClientAMProtocolPBServiceImpl.java | 70 + .../yarn/service/monitor/ServiceMonitor.java | 147 ++ .../yarn/service/monitor/probe/HttpProbe.java | 110 ++ .../service/monitor/probe/LogEntryBuilder.java | 76 + .../yarn/service/monitor/probe/MonitorKeys.java | 66 + .../service/monitor/probe/MonitorUtils.java | 84 + .../yarn/service/monitor/probe/PortProbe.java | 98 + .../yarn/service/monitor/probe/Probe.java | 100 + .../yarn/service/monitor/probe/ProbeStatus.java | 160 ++ .../provider/AbstractClientProvider.java | 129 ++ .../provider/AbstractProviderService.java | 113 ++ .../yarn/service/provider/ProviderFactory.java | 76 + .../yarn/service/provider/ProviderService.java | 39 + .../yarn/service/provider/ProviderUtils.java | 408 ++++ .../defaultImpl/DefaultClientProvider.java | 46 + .../defaultImpl/DefaultProviderFactory.java | 51 + .../defaultImpl/DefaultProviderService.java | 36 + .../provider/docker/DockerClientProvider.java | 53 + .../service/provider/docker/DockerKeys.java | 30 + .../provider/docker/DockerProviderFactory.java | 52 + .../provider/docker/DockerProviderService.java | 57 + .../provider/tarball/TarballClientProvider.java | 65 + .../tarball/TarballProviderFactory.java | 52 + .../tarball/TarballProviderService.java | 48 + .../registry/CustomRegistryConstants.java | 57 + .../registry/YarnRegistryViewForProviders.java | 249 +++ .../timelineservice/ServiceMetricsSink.java | 102 + .../ServiceTimelineEntityType.java | 39 + .../timelineservice/ServiceTimelineEvent.java | 36 + .../ServiceTimelineMetricsConstants.java | 92 + .../ServiceTimelinePublisher.java | 381 ++++ .../service/timelineservice/package-info.java | 27 + .../utils/ApplicationReportSerDeser.java | 56 + .../service/utils/ClientRegistryBinder.java | 201 ++ .../hadoop/yarn/service/utils/Comparators.java | 62 + .../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++ .../hadoop/yarn/service/utils/ConfigUtils.java | 97 + .../yarn/service/utils/CoreFileSystem.java | 521 ++++++ .../hadoop/yarn/service/utils/Duration.java | 109 ++ .../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++ .../yarn/service/utils/PatternValidator.java | 58 + .../hadoop/yarn/service/utils/PortScanner.java | 113 ++ .../service/utils/PublishedConfiguration.java | 196 ++ .../utils/PublishedConfigurationOutputter.java | 212 +++ .../utils/SerializedApplicationReport.java | 98 + .../yarn/service/utils/ServiceApiUtil.java | 401 ++++ .../service/utils/ServiceRegistryUtils.java | 71 + .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++ .../yarn/service/utils/SliderFileSystem.java | 51 + .../yarn/service/utils/ZookeeperUtils.java | 146 ++ .../src/main/proto/ClientAMProtocol.proto | 56 + .../hadoop/yarn/service/MockServiceAM.java | 221 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 262 +++ .../hadoop/yarn/service/TestServiceApiUtil.java | 480 +++++ .../yarn/service/TestYarnNativeServices.java | 403 ++++ .../client/TestBuildExternalComponents.java | 119 ++ .../yarn/service/client/TestServiceCLI.java | 127 ++ .../yarn/service/conf/ExampleAppJson.java | 65 + .../yarn/service/conf/TestAppJsonResolve.java | 200 ++ .../service/conf/TestLoadExampleAppJson.java | 71 + .../service/conf/TestValidateServiceNames.java | 125 ++ .../service/monitor/TestServiceMonitor.java | 104 ++ .../providers/TestAbstractClientProvider.java | 118 ++ .../service/providers/TestProviderFactory.java | 76 + .../TestServiceTimelinePublisher.java | 291 +++ .../src/test/resources/example-app.json | 15 + .../service/conf/examples/app-override.json | 76 + .../hadoop/yarn/service/conf/examples/app.json | 50 + .../yarn/service/conf/examples/default.json | 16 + .../yarn/service/conf/examples/external0.json | 15 + .../yarn/service/conf/examples/external1.json | 30 + .../yarn/service/conf/examples/external2.json | 22 + .../src/test/resources/yarn-site.xml | 19 + .../hadoop-yarn-services/pom.xml | 38 + .../hadoop-yarn-applications/pom.xml | 2 + .../hadoop/yarn/client/api/AppAdminClient.java | 222 +++ .../hadoop/yarn/client/cli/ApplicationCLI.java | 318 +++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 +- .../yarn/security/ContainerTokenIdentifier.java | 20 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 36 + .../src/main/proto/yarn_security_token.proto | 1 + .../src/main/resources/yarn-default.xml | 8 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 78 +- .../registry/client/api/DNSOperations.java | 60 + .../client/api/DNSOperationsFactory.java | 78 + .../registry/client/api/RegistryConstants.java | 136 +- .../client/api/RegistryOperationsFactory.java | 8 + .../client/binding/RegistryTypeUtils.java | 3 +- .../registry/client/impl/zk/CuratorService.java | 271 ++- .../registry/client/impl/zk/ListenerHandle.java | 25 + .../registry/client/impl/zk/PathListener.java | 30 + .../impl/zk/RegistryOperationsService.java | 4 +- .../types/yarn/YarnRegistryAttributes.java | 16 +- .../dns/ApplicationServiceRecordProcessor.java | 366 ++++ .../server/dns/BaseServiceRecordProcessor.java | 468 +++++ .../dns/ContainerServiceRecordProcessor.java | 283 +++ .../dns/PrivilegedRegistryDNSStarter.java | 80 + .../server/dns/RecordCreatorFactory.java | 275 +++ .../hadoop/registry/server/dns/RegistryDNS.java | 1756 ++++++++++++++++++ .../registry/server/dns/RegistryDNSServer.java | 267 +++ .../registry/server/dns/ReverseZoneUtils.java | 171 ++ .../registry/server/dns/SecureableZone.java | 151 ++ .../server/dns/ServiceRecordProcessor.java | 53 + .../registry/server/dns/ZoneSelector.java | 33 + .../registry/server/dns/package-info.java | 26 + .../registry/server/dns/TestRegistryDNS.java | 627 +++++++ .../server/dns/TestReverseZoneUtils.java | 89 + .../server/dns/TestSecureRegistryDNS.java | 44 + .../test/resources/0.17.172.in-addr.arpa.zone | 36 + .../src/test/resources/test.private | 32 + .../api/protocolrecords/NMContainerStatus.java | 18 +- .../impl/pb/NMContainerStatusPBImpl.java | 15 +- .../OpportunisticContainerAllocator.java | 2 +- .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 5 +- .../container/ContainerImpl.java | 8 +- .../runtime/DockerLinuxContainerRuntime.java | 8 + .../blacklist/SimpleBlacklistManager.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 35 +- .../scheduler/AbstractYarnScheduler.java | 1 + .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../security/RMContainerTokenSecretManager.java | 36 +- .../server/resourcemanager/webapp/RMWebApp.java | 19 + .../server/resourcemanager/TestRMRestart.java | 2 +- .../TestResourceTrackerService.java | 6 +- .../capacity/TestContainerAllocation.java | 5 +- .../src/site/markdown/YarnCommands.md | 33 +- .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 4 +- .../src/site/markdown/yarn-service/Concepts.md | 46 + .../markdown/yarn-service/Configurations.md | 165 ++ .../src/site/markdown/yarn-service/Examples.md | 161 ++ .../src/site/markdown/yarn-service/Overview.md | 61 + .../site/markdown/yarn-service/QuickStart.md | 217 +++ .../site/markdown/yarn-service/RegistryDNS.md | 200 ++ .../markdown/yarn-service/ServiceDiscovery.md | 136 ++ .../markdown/yarn-service/YarnServiceAPI.md | 598 ++++++ .../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes .../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes .../resources/images/dns_record_removal.jpeg | Bin 0 -> 58041 bytes .../main/webapp/app/adapters/restabstract.js | 50 + .../app/adapters/yarn-component-instance.js | 32 + .../app/adapters/yarn-service-component.js | 32 + .../webapp/app/adapters/yarn-service-info.js | 32 + .../main/webapp/app/adapters/yarn-servicedef.js | 44 + .../webapp/app/components/breadcrumb-bar.js | 1 + .../webapp/app/components/confirm-dialog.js | 31 + .../webapp/app/components/deploy-service.js | 167 ++ .../app/components/fileconfig-viewer-dialog.js | 36 + .../main/webapp/app/components/info-tooltip.js | 44 + .../main/webapp/app/components/metrics-table.js | 24 + .../app/components/service-component-table.js | 56 + .../app/components/service-config-table.js | 89 + .../app/components/service-fileconfig-table.js | 112 ++ .../main/webapp/app/components/upload-config.js | 54 + .../app/controllers/yarn-app/components.js | 59 + .../webapp/app/controllers/yarn-app/configs.js | 24 + .../webapp/app/controllers/yarn-app/info.js | 60 + .../app/controllers/yarn-component-instance.js | 59 + .../controllers/yarn-component-instance/info.js | 25 + .../app/controllers/yarn-component-instances.js | 59 + .../yarn-component-instances/configs.js | 25 + .../yarn-component-instances/info.js | 62 + .../app/controllers/yarn-deploy-service.js | 69 + .../webapp/app/helpers/check-availability.js | 28 + .../app/models/yarn-component-instance.js | 51 + .../webapp/app/models/yarn-service-component.js | 46 + .../main/webapp/app/models/yarn-service-info.js | 57 + .../main/webapp/app/models/yarn-servicedef.js | 278 +++ .../src/main/webapp/app/router.js | 10 + .../webapp/app/routes/yarn-app/components.js | 49 + .../main/webapp/app/routes/yarn-app/configs.js | 52 + .../src/main/webapp/app/routes/yarn-app/info.js | 12 +- .../app/routes/yarn-component-instance.js | 29 + .../app/routes/yarn-component-instance/info.js | 45 + .../app/routes/yarn-component-instances.js | 29 + .../routes/yarn-component-instances/configs.js | 44 + .../app/routes/yarn-component-instances/info.js | 53 + .../webapp/app/routes/yarn-deploy-service.js | 27 + .../src/main/webapp/app/routes/yarn-services.js | 2 +- .../app/serializers/yarn-component-instance.js | 72 + .../app/serializers/yarn-service-component.js | 77 + .../webapp/app/serializers/yarn-service-info.js | 87 + .../src/main/webapp/app/services/hosts.js | 4 + .../src/main/webapp/app/styles/app.css | 176 ++ .../main/webapp/app/templates/application.hbs | 2 + .../app/templates/components/breadcrumb-bar.hbs | 4 +- .../app/templates/components/confirm-dialog.hbs | 37 + .../app/templates/components/deploy-service.hbs | 157 ++ .../components/fileconfig-viewer-dialog.hbs | 53 + .../app/templates/components/info-tooltip.hbs | 20 + .../app/templates/components/metrics-table.hbs | 55 + .../components/service-component-table.hbs | 113 ++ .../components/service-config-table.hbs | 130 ++ .../components/service-fileconfig-table.hbs | 152 ++ .../app/templates/components/upload-config.hbs | 44 + .../src/main/webapp/app/templates/yarn-app.hbs | 8 + .../app/templates/yarn-app/components.hbs | 23 + .../webapp/app/templates/yarn-app/configs.hbs | 57 + .../main/webapp/app/templates/yarn-app/info.hbs | 111 +- .../app/templates/yarn-component-instance.hbs | 43 + .../templates/yarn-component-instance/info.hbs | 77 + .../app/templates/yarn-component-instances.hbs | 46 + .../yarn-component-instances/configs.hbs | 53 + .../templates/yarn-component-instances/info.hbs | 28 + .../yarn-component-instances/loading.hbs | 23 + .../app/templates/yarn-deploy-service.hbs | 33 + .../main/webapp/app/templates/yarn-services.hbs | 4 + .../src/main/webapp/app/utils/info-seeder.js | 26 + .../src/main/webapp/config/default-config.js | 6 +- .../components/confirm-dialog-test.js | 43 + .../components/deploy-service-test.js | 43 + .../components/fileconfig-viewer-dialog-test.js | 43 + .../integration/components/info-tooltip-test.js | 43 + .../components/metrics-table-test.js | 43 + .../components/service-component-table-test.js | 43 + .../components/service-config-table-test.js | 43 + .../components/service-fileconfig-table-test.js | 43 + .../components/upload-config-test.js | 43 + .../adapters/yarn-component-instance-test.js | 30 + .../adapters/yarn-service-component-test.js | 30 + .../unit/adapters/yarn-service-info-test.js | 30 + .../tests/unit/adapters/yarn-servicedef-test.js | 30 + .../controllers/yarn-app/components-test.js | 30 + .../unit/controllers/yarn-app/configs-test.js | 30 + .../controllers/yarn-component-instance-test.js | 30 + .../yarn-component-instance/info-test.js | 30 + .../yarn-component-instances-test.js | 30 + .../yarn-component-instances/configs-test.js | 30 + .../yarn-component-instances/info-test.js | 30 + .../controllers/yarn-deploy-service-test.js | 30 + .../unit/helpers/check-availability-test.js | 28 + .../unit/models/yarn-component-instance-test.js | 30 + .../unit/models/yarn-service-component-test.js | 30 + .../tests/unit/models/yarn-service-info-test.js | 30 + .../tests/unit/models/yarn-servicedef-test.js | 30 + .../unit/routes/yarn-app/components-test.js | 29 + .../tests/unit/routes/yarn-app/configs-test.js | 29 + .../unit/routes/yarn-component-instance-test.js | 29 + .../routes/yarn-component-instance/info-test.js | 29 + .../routes/yarn-component-instances-test.js | 29 + .../yarn-component-instances/configs-test.js | 29 + .../yarn-component-instances/info-test.js | 29 + .../unit/routes/yarn-deploy-service-test.js | 29 + .../serializers/yarn-component-instance-test.js | 33 + .../serializers/yarn-service-component-test.js | 33 + .../unit/serializers/yarn-service-info-test.js | 33 + hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +- hadoop-yarn-project/pom.xml | 4 + 334 files changed, 33539 insertions(+), 320 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
