Repository: hadoop Updated Branches: refs/heads/HADOOP-13345-merge2 [created] 2965cd8be
Merge branch 'trunk' into HADOOP-13345-merge2 After HADOOP-14040, we use shaded aws sdk uber-JAR so we don't have to bring DynamoDB dependency explicitly. However, for tests we do need the DynamoDBLocal dependency from its Maven repository. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2965cd8b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2965cd8b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2965cd8b Branch: refs/heads/HADOOP-13345-merge2 Commit: 2965cd8beceb39217c8dde4f1e2e788fe01bddb6 Parents: 2ff5812 0013090 Author: Mingliang Liu <[email protected]> Authored: Wed Feb 22 13:48:30 2017 -0800 Committer: Mingliang Liu <[email protected]> Committed: Wed Feb 22 14:51:19 2017 -0800 ---------------------------------------------------------------------- .gitignore | 12 +- .../resources/assemblies/hadoop-httpfs-dist.xml | 25 +- .../main/resources/checkstyle/checkstyle.xml | 4 +- hadoop-client-modules/hadoop-client-api/pom.xml | 12 +- .../hadoop-client-minicluster/pom.xml | 22 +- .../hadoop-client-runtime/pom.xml | 16 +- .../org/apache/hadoop/conf/Configuration.java | 12 + .../hadoop/crypto/key/CachingKeyProvider.java | 14 +- .../apache/hadoop/crypto/key/KeyProvider.java | 12 + .../hadoop/crypto/key/KeyProviderExtension.java | 5 + .../org/apache/hadoop/crypto/key/KeyShell.java | 67 +- .../crypto/key/kms/KMSClientProvider.java | 13 +- .../hadoop/crypto/key/kms/KMSRESTConstants.java | 1 + .../key/kms/LoadBalancingKMSClientProvider.java | 24 +- .../hadoop/crypto/key/kms/ValueQueue.java | 117 ++- .../org/apache/hadoop/fs/BlockLocation.java | 7 +- .../hadoop/fs/CommonConfigurationKeys.java | 11 + .../apache/hadoop/fs/FSExceptionMessages.java | 3 + .../java/org/apache/hadoop/fs/FileStatus.java | 21 +- .../hadoop/fs/permission/FsCreateModes.java | 1 + .../hadoop/fs/permission/FsPermission.java | 22 +- .../java/org/apache/hadoop/fs/shell/Stat.java | 4 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 37 +- .../org/apache/hadoop/http/HttpServer2.java | 119 ++- .../hadoop/io/retry/RetryInvocationHandler.java | 8 +- .../main/java/org/apache/hadoop/ipc/Client.java | 6 +- .../org/apache/hadoop/ipc/FairCallQueue.java | 165 +--- .../main/java/org/apache/hadoop/ipc/Server.java | 206 ++-- .../java/org/apache/hadoop/net/InnerNode.java | 67 ++ .../org/apache/hadoop/net/InnerNodeImpl.java | 304 ++++++ .../org/apache/hadoop/net/NetworkTopology.java | 326 +----- .../net/NetworkTopologyWithNodeGroup.java | 43 +- .../AuthenticationWithProxyUserFilter.java | 43 +- .../hadoop/security/UserGroupInformation.java | 129 ++- .../ZKDelegationTokenSecretManager.java | 33 +- .../src/site/markdown/CommandsManual.md | 3 + .../src/site/markdown/CredentialProviderAPI.md | 1 + .../src/site/markdown/FileSystemShell.md | 2 +- .../hadoop-common/src/site/markdown/Metrics.md | 6 + .../src/site/markdown/SecureMode.md | 2 +- .../hadoop-common/src/site/markdown/Tracing.md | 2 +- .../apache/hadoop/crypto/key/TestKeyShell.java | 9 + .../org/apache/hadoop/fs/TestFileStatus.java | 19 + .../hadoop/ha/TestActiveStandbyElector.java | 31 + .../org/apache/hadoop/http/TestHttpServer.java | 38 + .../hadoop/http/TestHttpServerWithSpengo.java | 15 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 57 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 119 +++ .../hadoop/security/TestUGIWithMiniKdc.java | 1 + .../security/TestUserGroupInformation.java | 16 +- .../src/test/resources/testConf.xml | 2 +- ...rKeyGeneratorKeyProviderCryptoExtension.java | 6 + .../hadoop/crypto/key/kms/server/KMS.java | 33 +- .../crypto/key/kms/server/KMSConfiguration.java | 2 + .../hadoop/crypto/key/kms/server/KMSWebApp.java | 10 - .../crypto/key/kms/server/KMSWebServer.java | 3 + .../kms/server/KeyAuthorizationKeyProvider.java | 11 + .../src/main/resources/kms-default.xml | 14 + .../hadoop-kms/src/site/markdown/index.md.vm | 52 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 89 +- .../crypto/key/kms/server/TestKMSAudit.java | 2 + .../org/apache/hadoop/fs/HdfsBlockLocation.java | 17 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 48 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 105 +- .../hadoop/hdfs/DistributedFileSystem.java | 2 +- .../apache/hadoop/hdfs/StripedDataStreamer.java | 8 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 6 +- .../hdfs/protocol/FsPermissionExtension.java | 2 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 51 + .../token/block/BlockTokenIdentifier.java | 89 +- .../src/main/proto/hdfs.proto | 33 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 103 +- .../src/main/conf/httpfs-env.sh | 47 +- .../http/server/HttpFSAuthenticationFilter.java | 2 +- .../fs/http/server/HttpFSServerWebServer.java | 170 ++++ .../apache/hadoop/lib/servlet/MDCFilter.java | 4 +- .../src/main/libexec/httpfs-config.sh | 76 -- .../libexec/shellprofile.d/hadoop-httpfs.sh | 67 ++ .../src/main/resources/httpfs-default.xml | 72 ++ .../main/resources/webapps/static/index.html | 36 + .../resources/webapps/webhdfs/WEB-INF/web.xml | 98 ++ .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 126 +-- .../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 - .../src/main/tomcat/ROOT/index.html | 21 - .../src/main/tomcat/logging.properties | 67 -- .../src/main/tomcat/server.xml | 151 --- .../src/main/tomcat/ssl-server.xml.conf | 136 --- .../src/site/markdown/ServerSetup.md.vm | 136 ++- .../src/site/markdown/index.md | 2 +- .../http/server/TestHttpFSServerWebServer.java | 106 ++ .../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 6 +- .../hadoop-hdfs/src/main/bin/hdfs | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 20 +- .../hadoop/hdfs/qjournal/server/JNStorage.java | 7 + .../token/block/BlockTokenSecretManager.java | 18 +- .../hadoop/hdfs/server/balancer/Balancer.java | 2 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 3 +- .../hadoop/hdfs/server/balancer/KeyManager.java | 6 +- .../server/blockmanagement/BlockManager.java | 9 +- .../BlockUnderConstructionFeature.java | 25 + .../blockmanagement/DatanodeDescriptor.java | 2 +- .../server/blockmanagement/DatanodeManager.java | 2 +- .../PendingReconstructionBlocks.java | 4 +- .../hadoop/hdfs/server/common/Storage.java | 2 +- .../hdfs/server/datanode/BlockReceiver.java | 3 +- .../hadoop/hdfs/server/datanode/DNConf.java | 12 +- .../hadoop/hdfs/server/datanode/DataNode.java | 15 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 3 + .../datanode/web/webhdfs/WebHdfsHandler.java | 6 + .../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +- .../hdfs/server/namenode/FSEditLogOp.java | 29 +- .../server/namenode/FSPermissionChecker.java | 87 +- .../hadoop/hdfs/server/namenode/NameNode.java | 112 ++- .../hdfs/server/namenode/NamenodeFsck.java | 28 +- .../protocol/BalancerBandwidthCommand.java | 2 +- .../apache/hadoop/hdfs/tools/AdminHelper.java | 12 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 23 +- .../hadoop/hdfs/tools/erasurecode/ECCli.java | 2 +- .../src/main/resources/hdfs-default.xml | 11 +- .../src/main/webapps/datanode/datanode.html | 2 + .../src/main/webapps/static/hadoop.css | 9 +- .../src/site/markdown/HDFSCommands.md | 12 +- .../src/site/markdown/HDFSErasureCoding.md | 4 +- .../src/site/markdown/HdfsImageViewer.md | 63 ++ .../hadoop/fs/permission/TestStickyBit.java | 63 ++ .../apache/hadoop/hdfs/TestDFSOutputStream.java | 3 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 24 +- .../hadoop/hdfs/TestMaintenanceState.java | 6 + .../java/org/apache/hadoop/hdfs/TestPread.java | 51 + .../hdfs/qjournal/server/TestJournal.java | 17 + .../security/token/block/TestBlockToken.java | 297 +++++- .../hdfs/server/datanode/DataNodeTestUtils.java | 26 + .../datanode/TestDataNodeHotSwapVolumes.java | 9 +- .../datanode/TestDataNodeVolumeFailure.java | 65 +- .../TestDataNodeVolumeFailureReporting.java | 12 +- .../TestDataNodeVolumeFailureToleration.java | 6 + .../shortcircuit/TestShortCircuitLocalRead.java | 2 +- .../hdfs/tools/TestStoragePolicyCommands.java | 14 +- .../TestOfflineImageViewer.java | 11 + .../apache/hadoop/net/TestNetworkTopology.java | 2 +- .../test/resources/testErasureCodingConf.xml | 2 +- .../hadoop/mapred/TaskAttemptListenerImpl.java | 17 +- .../jobhistory/JobHistoryEventHandler.java | 57 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +- .../v2/app/client/MRClientService.java | 4 +- .../v2/app/rm/RMContainerAllocator.java | 4 +- .../mapreduce/v2/app/webapp/AppController.java | 7 +- .../jobhistory/TestJobHistoryEventHandler.java | 8 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 +- .../src/main/resources/mapred-default.xml | 8 + .../site/markdown/DistributedCacheDeploy.md.vm | 2 + .../src/site/markdown/EncryptedShuffle.md | 2 + .../src/site/markdown/MapReduceTutorial.md | 45 +- .../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 2 + .../src/site/markdown/MapredAppMasterRest.md | 16 +- .../src/site/markdown/MapredCommands.md | 14 +- .../PluggableShuffleAndPluggableSort.md | 2 + .../src/site/markdown/HistoryServerRest.md | 22 +- .../hadoop/mapred/ResourceMgrDelegate.java | 6 +- .../org/apache/hadoop/examples/pi/package.html | 4 +- hadoop-project/pom.xml | 27 +- hadoop-project/src/site/site.xml | 4 +- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 5 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 47 +- .../fs/aliyun/oss/AliyunOSSInputStream.java | 4 +- .../oss/TestAliyunOSSFileSystemContract.java | 7 + .../fs/aliyun/oss/TestAliyunOSSInputStream.java | 9 + .../oss/contract/TestAliyunOSSContractSeek.java | 26 + .../src/test/resources/contract/aliyun-oss.xml | 5 + hadoop-tools/hadoop-aws/pom.xml | 19 +- .../s3a/BlockingThreadPoolExecutorService.java | 2 +- .../org/apache/hadoop/fs/s3a/Constants.java | 21 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 1 + .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 61 ++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 152 ++- .../apache/hadoop/fs/s3a/S3AInputStream.java | 24 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 10 + .../hadoop/fs/s3a/S3ObjectAttributes.java | 59 ++ .../src/site/markdown/tools/hadoop-aws/index.md | 579 +---------- .../site/markdown/tools/hadoop-aws/testing.md | 814 +++++++++++++++ .../fs/s3a/AbstractTestS3AEncryption.java | 132 +++ .../hadoop/fs/s3a/ITestS3AEncryption.java | 97 -- .../ITestS3AEncryptionAlgorithmPropagation.java | 76 -- .../ITestS3AEncryptionAlgorithmValidation.java | 152 +++ .../ITestS3AEncryptionBlockOutputStream.java | 36 - .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 90 ++ ...ITestS3AEncryptionSSECBlockOutputStream.java | 46 + .../s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 57 ++ .../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 48 + ...onSSEKMSUserDefinedKeyBlockOutputStream.java | 52 + .../hadoop/fs/s3a/ITestS3AEncryptionSSES3.java | 43 + ...TestS3AEncryptionSSES3BlockOutputStream.java | 45 + .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 57 +- .../fs/s3a/scale/ITestS3AConcurrentOps.java | 167 ++++ .../NativeS3FileSystemContractBaseTest.java | 2 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 4 + .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 31 +- .../src/site/markdown/index.md | 30 +- .../hadoop/fs/adl/TestADLResponseData.java | 21 + .../apache/hadoop/fs/adl/TestGetFileStatus.java | 25 + .../apache/hadoop/fs/adl/TestListStatus.java | 32 + .../fs/adl/TestValidateConfiguration.java | 9 + .../fs/adl/live/AdlStorageConfiguration.java | 42 +- .../apache/hadoop/fs/adl/live/TestMetadata.java | 33 + .../src/test/resources/adls.xml | 11 + .../test/resources/contract-test-options.xml | 61 -- .../fs/swift/auth/PasswordCredentials.java | 3 +- .../hadoop/yarn/api/ApplicationConstants.java | 7 - .../GetLabelsToNodesResponse.java | 6 +- .../GetNodesToLabelsResponse.java | 7 +- .../yarn/api/records/UpdateContainerError.java | 19 +- .../hadoop/yarn/conf/YarnConfiguration.java | 10 +- .../AddToClusterNodeLabelsRequest.java | 11 +- ..._server_resourcemanager_service_protos.proto | 5 +- .../src/main/proto/yarn_protos.proto | 6 +- .../src/main/proto/yarn_service_protos.proto | 6 +- .../distributedshell/ApplicationMaster.java | 98 +- .../hadoop/yarn/client/api/AMRMClient.java | 71 +- .../hadoop/yarn/client/api/YarnClient.java | 6 +- .../yarn/client/api/async/AMRMClientAsync.java | 52 +- .../api/async/impl/AMRMClientAsyncImpl.java | 15 +- .../yarn/client/api/impl/AMRMClientImpl.java | 111 ++- .../yarn/client/api/impl/YarnClientImpl.java | 83 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 17 - .../hadoop/yarn/client/TestRMFailover.java | 100 +- .../yarn/client/api/impl/TestAMRMClient.java | 60 +- .../api/impl/TestAMRMClientOnRMRestart.java | 8 +- .../TestOpportunisticContainerAllocation.java | 400 +++++++- .../yarn/client/api/impl/TestYarnClient.java | 216 +++- .../pb/GetClusterNodeLabelsResponsePBImpl.java | 4 + .../impl/pb/GetLabelsToNodesResponsePBImpl.java | 20 +- .../impl/pb/GetNodesToLabelsResponsePBImpl.java | 52 +- .../impl/pb/UpdateContainerErrorPBImpl.java | 16 + .../RequestHedgingRMFailoverProxyProvider.java | 6 +- .../hadoop/yarn/client/api/TimelineClient.java | 94 +- .../yarn/client/api/TimelineV2Client.java | 92 ++ .../client/api/impl/TimelineClientImpl.java | 825 ++-------------- .../yarn/client/api/impl/TimelineConnector.java | 440 +++++++++ .../client/api/impl/TimelineV2ClientImpl.java | 459 +++++++++ .../ContainerLogAggregationType.java | 31 + .../yarn/logaggregation/ContainerLogType.java | 31 - .../yarn/logaggregation/LogToolUtils.java | 159 +++ .../pb/AddToClusterNodeLabelsRequestPBImpl.java | 47 +- .../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +- .../hadoop/yarn/util/resource/Resources.java | 24 + .../org/apache/hadoop/yarn/webapp/WebApps.java | 38 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 31 +- .../yarn/webapp/util/YarnWebServiceUtils.java | 62 ++ .../src/main/resources/yarn-default.xml | 23 + .../client/api/impl/TestTimelineClient.java | 39 +- .../api/impl/TestTimelineClientV2Impl.java | 4 +- .../apache/hadoop/yarn/webapp/TestWebApp.java | 46 + .../webapp/AHSWebServices.java | 354 +++---- .../webapp/TestAHSWebServices.java | 157 ++- .../hadoop/yarn/server/webapp/AppBlock.java | 135 +-- .../server/webapp/YarnWebServiceParams.java | 37 + .../server/webapp/dao/ContainerLogsInfo.java | 8 +- .../timelineservice/NMTimelinePublisher.java | 22 +- .../nodemanager/webapp/NMWebServices.java | 170 +++- .../webapp/dao/NMContainerLogsInfo.java | 4 +- .../TestNMTimelinePublisher.java | 10 +- .../nodemanager/webapp/TestNMWebServices.java | 161 ++- .../hadoop-yarn-server-resourcemanager/pom.xml | 16 +- .../server/resourcemanager/AdminService.java | 7 +- .../server/resourcemanager/ClientRMService.java | 6 +- .../DecommissioningNodesWatcher.java | 4 +- .../yarn/server/resourcemanager/RMContext.java | 2 + .../server/resourcemanager/RMContextImpl.java | 10 + ...MCriticalThreadUncaughtExceptionHandler.java | 58 ++ .../resourcemanager/RMFatalEventType.java | 5 +- .../server/resourcemanager/RMServerUtils.java | 36 +- .../server/resourcemanager/ResourceManager.java | 65 +- .../resourcemanager/amlauncher/AMLauncher.java | 13 - .../FifoIntraQueuePreemptionPlugin.java | 6 +- .../nodelabels/RMNodeLabelsManager.java | 16 + .../resourcemanager/recovery/RMStateStore.java | 13 +- .../recovery/ZKRMStateStore.java | 27 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 12 +- .../rmapp/attempt/RMAppAttemptImpl.java | 176 +++- .../scheduler/AbstractUsersManager.java | 54 + .../scheduler/AbstractYarnScheduler.java | 4 + .../scheduler/ActiveUsersManager.java | 23 +- .../scheduler/AppSchedulingInfo.java | 16 +- .../server/resourcemanager/scheduler/Queue.java | 2 +- .../scheduler/SchedulerApplicationAttempt.java | 4 +- .../scheduler/capacity/CSQueue.java | 8 +- .../capacity/CapacityHeadroomProvider.java | 4 +- .../scheduler/capacity/CapacityScheduler.java | 3 +- .../scheduler/capacity/LeafQueue.java | 589 +++-------- .../scheduler/capacity/ParentQueue.java | 2 +- .../scheduler/capacity/UsersManager.java | 982 +++++++++++++++++++ .../scheduler/common/fica/FiCaSchedulerApp.java | 14 +- .../scheduler/fair/AllocationConfiguration.java | 11 +- .../scheduler/fair/FSAppAttempt.java | 185 +++- .../scheduler/fair/FSLeafQueue.java | 122 ++- .../scheduler/fair/FSOpDurations.java | 8 - .../scheduler/fair/FSParentQueue.java | 15 +- .../scheduler/fair/FSPreemptionThread.java | 132 +-- .../resourcemanager/scheduler/fair/FSQueue.java | 50 +- .../scheduler/fair/FairScheduler.java | 30 +- .../fair/FairSchedulerConfiguration.java | 23 +- .../scheduler/fair/QueueManager.java | 28 +- .../scheduler/fair/SchedulingPolicy.java | 36 +- .../fair/VisitedResourceRequestTracker.java | 146 +++ .../DominantResourceFairnessPolicy.java | 5 - .../fair/policies/FairSharePolicy.java | 15 +- .../scheduler/fair/policies/FifoPolicy.java | 14 +- .../scheduler/fifo/FifoScheduler.java | 2 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 2 +- .../TestApplicationMasterLauncher.java | 5 - .../resourcemanager/TestClientRMService.java | 42 +- ...pportunisticContainerAllocatorAMService.java | 5 +- ...alCapacityPreemptionPolicyMockFramework.java | 8 +- .../rmapp/attempt/TestBoundedAppender.java | 116 +++ .../TestRMAppAttemptImplDiagnostics.java | 111 +++ .../TestSchedulerApplicationAttempt.java | 14 +- .../capacity/TestApplicationLimits.java | 15 +- .../TestApplicationLimitsByPartition.java | 23 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 77 +- .../capacity/TestIncreaseAllocationExpirer.java | 4 +- .../scheduler/capacity/TestLeafQueue.java | 98 +- .../TestNodeLabelContainerAllocation.java | 7 +- .../fair/FairSchedulerWithMockPreemption.java | 5 +- .../fair/TestContinuousScheduling.java | 2 +- .../scheduler/fair/TestFSAppStarvation.java | 30 +- .../scheduler/fair/TestFairScheduler.java | 72 +- .../fair/TestFairSchedulerPreemption.java | 73 +- .../scheduler/fair/TestSchedulingPolicy.java | 302 ++++-- .../fair/TestVisitedResourceRequestTracker.java | 112 +++ .../TestTimelineServiceClientIntegration.java | 10 +- .../reader/TimelineReaderServer.java | 13 +- .../src/site/markdown/CapacityScheduler.md | 14 +- .../src/site/markdown/DockerContainers.md | 293 ++++++ .../src/site/markdown/FairScheduler.md | 14 +- .../src/site/markdown/NodeLabel.md | 14 +- .../src/site/markdown/NodeManager.md | 8 +- .../src/site/markdown/NodeManagerCgroups.md | 3 +- .../src/site/markdown/NodeManagerRest.md | 116 ++- .../site/markdown/OpportunisticContainers.md | 17 +- .../src/site/markdown/ReservationSystem.md | 6 +- .../src/site/markdown/ResourceManagerHA.md | 10 +- .../src/site/markdown/ResourceManagerRest.md | 29 +- .../src/site/markdown/ResourceManagerRestart.md | 13 +- .../src/site/markdown/SecureContainer.md | 2 +- .../src/site/markdown/TimelineServer.md | 16 +- .../src/site/markdown/TimelineServiceV2.md | 154 ++- .../src/site/markdown/WebApplicationProxy.md | 7 +- .../src/site/markdown/WebServicesIntro.md | 13 +- .../site/markdown/WritingYarnApplications.md | 14 +- .../site/markdown/YarnApplicationSecurity.md | 2 + .../src/site/markdown/YarnCommands.md | 26 +- .../hadoop-yarn-ui/src/main/webapp/.jshintrc | 5 +- .../src/main/webapp/app/adapters/abstract.js | 1 + .../main/webapp/app/adapters/cluster-info.js | 4 +- .../main/webapp/app/adapters/cluster-metric.js | 4 +- .../webapp/app/adapters/yarn-app-attempt.js | 4 +- .../src/main/webapp/app/adapters/yarn-app.js | 6 +- .../webapp/app/adapters/yarn-container-log.js | 4 +- .../main/webapp/app/adapters/yarn-container.js | 12 +- .../src/main/webapp/app/adapters/yarn-node.js | 4 +- .../src/main/webapp/app/adapters/yarn-queue.js | 4 +- .../main/webapp/app/adapters/yarn-rm-node.js | 6 +- .../webapp/app/components/app-attempt-table.js | 9 +- .../app/components/app-usage-donut-chart.js | 6 +- .../src/main/webapp/app/components/bar-chart.js | 36 +- .../app/components/base-chart-component.js | 8 +- .../app/components/base-usage-donut-chart.js | 11 +- .../main/webapp/app/components/donut-chart.js | 10 +- .../main/webapp/app/components/nodes-heatmap.js | 21 +- ...er-app-memusage-by-nodes-stacked-barchart.js | 4 +- ...app-ncontainers-by-nodes-stacked-barchart.js | 4 +- .../app/components/queue-usage-donut-chart.js | 4 +- .../main/webapp/app/components/queue-view.js | 11 +- .../main/webapp/app/components/simple-table.js | 6 +- .../webapp/app/components/stacked-barchart.js | 23 +- .../main/webapp/app/components/timeline-view.js | 25 +- .../main/webapp/app/components/tree-selector.js | 41 +- .../main/webapp/app/controllers/application.js | 8 +- .../webapp/app/controllers/yarn-app-attempt.js | 2 +- .../webapp/app/controllers/yarn-app-attempts.js | 2 +- .../src/main/webapp/app/controllers/yarn-app.js | 8 +- .../main/webapp/app/controllers/yarn-apps.js | 2 +- .../app/controllers/yarn-container-log.js | 7 +- .../webapp/app/controllers/yarn-node-app.js | 7 +- .../webapp/app/controllers/yarn-node-apps.js | 2 +- .../app/controllers/yarn-node-container.js | 39 + .../app/controllers/yarn-node-containers.js | 2 +- .../main/webapp/app/controllers/yarn-node.js | 2 +- .../webapp/app/controllers/yarn-queue-apps.js | 46 - .../main/webapp/app/controllers/yarn-queue.js | 17 +- .../main/webapp/app/controllers/yarn-queues.js | 1 + .../webapp/app/controllers/yarn-services.js | 2 +- .../src/main/webapp/app/helpers/divide.js | 2 +- .../main/webapp/app/helpers/log-files-comma.js | 5 +- .../src/main/webapp/app/helpers/node-link.js | 2 +- .../src/main/webapp/app/helpers/node-menu.js | 10 +- .../src/main/webapp/app/helpers/node-name.js | 4 +- .../src/main/webapp/app/initializers/loader.js | 8 +- .../main/webapp/app/models/yarn-app-attempt.js | 17 +- .../src/main/webapp/app/models/yarn-app.js | 10 +- .../src/main/webapp/app/models/yarn-node-app.js | 8 +- .../webapp/app/models/yarn-node-container.js | 12 +- .../src/main/webapp/app/models/yarn-queue.js | 14 +- .../src/main/webapp/app/models/yarn-rm-node.js | 10 +- .../src/main/webapp/app/models/yarn-user.js | 2 +- .../src/main/webapp/app/router.js | 6 +- .../src/main/webapp/app/routes/application.js | 3 +- .../main/webapp/app/routes/cluster-overview.js | 4 +- .../main/webapp/app/routes/yarn-app-attempt.js | 12 +- .../main/webapp/app/routes/yarn-app-attempts.js | 2 - .../src/main/webapp/app/routes/yarn-app.js | 2 +- .../src/main/webapp/app/routes/yarn-apps.js | 4 +- .../webapp/app/routes/yarn-container-log.js | 2 +- .../src/main/webapp/app/routes/yarn-node-app.js | 2 +- .../webapp/app/routes/yarn-node-container.js | 2 +- .../src/main/webapp/app/routes/yarn-node.js | 4 +- .../src/main/webapp/app/routes/yarn-nodes.js | 4 +- .../main/webapp/app/routes/yarn-queue-apps.js | 42 - .../src/main/webapp/app/routes/yarn-queue.js | 3 +- .../main/webapp/app/routes/yarn-queue/apps.js | 22 + .../main/webapp/app/routes/yarn-queue/info.js | 22 + .../src/main/webapp/app/routes/yarn-queues.js | 35 +- .../main/webapp/app/routes/yarn-queues/index.js | 2 + .../webapp/app/serializers/yarn-app-attempt.js | 23 +- .../src/main/webapp/app/serializers/yarn-app.js | 19 +- .../app/serializers/yarn-container-log.js | 3 +- .../webapp/app/serializers/yarn-container.js | 22 +- .../webapp/app/serializers/yarn-node-app.js | 24 +- .../app/serializers/yarn-node-container.js | 7 +- .../main/webapp/app/serializers/yarn-node.js | 8 +- .../main/webapp/app/serializers/yarn-queue.js | 10 +- .../main/webapp/app/serializers/yarn-rm-node.js | 7 +- .../src/main/webapp/app/styles/app.css | 28 +- .../main/webapp/app/templates/application.hbs | 8 +- .../webapp/app/templates/cluster-overview.hbs | 10 +- .../templates/components/app-attempt-table.hbs | 6 +- .../templates/components/container-table.hbs | 6 +- .../src/main/webapp/app/templates/loading.hbs | 23 + .../src/main/webapp/app/templates/yarn-app.hbs | 8 +- .../webapp/app/templates/yarn-apps/loading.hbs | 23 + .../webapp/app/templates/yarn-queue-apps.hbs | 64 -- .../main/webapp/app/templates/yarn-queue.hbs | 69 +- .../webapp/app/templates/yarn-queue/apps.hbs | 28 + .../webapp/app/templates/yarn-queue/info.hbs | 84 ++ .../main/webapp/app/templates/yarn-queues.hbs | 8 +- .../src/main/webapp/app/utils/color-utils.js | 6 +- .../src/main/webapp/app/utils/converter.js | 2 +- .../main/webapp/app/utils/href-address-utils.js | 2 - .../src/main/webapp/app/utils/mock.js | 4 +- .../src/main/webapp/app/utils/sorter.js | 8 +- .../src/main/webapp/ember-cli-build.js | 2 +- .../webapp/public/assets/images/spinner.gif | Bin 0 -> 33076 bytes .../controllers/yarn-node-container-test.js | 30 + .../unit/controllers/yarn-queue-apps-test.js | 30 - .../unit/models/yarn-container-log-test.js | 1 + .../tests/unit/models/yarn-node-app-test.js | 1 + .../unit/models/yarn-node-container-test.js | 1 + .../webapp/tests/unit/models/yarn-node-test.js | 1 + .../tests/unit/models/yarn-rm-node-test.js | 1 + .../unit/routes/yarn-container-log-test.js | 24 +- .../tests/unit/routes/yarn-node-app-test.js | 28 +- .../tests/unit/routes/yarn-node-apps-test.js | 30 +- .../unit/routes/yarn-node-container-test.js | 28 +- .../unit/routes/yarn-node-containers-test.js | 30 +- .../webapp/tests/unit/routes/yarn-node-test.js | 4 +- .../webapp/tests/unit/routes/yarn-nodes-test.js | 2 +- .../tests/unit/routes/yarn-queue-apps-test.js | 29 - .../tests/unit/routes/yarn-queue/apps-test.js | 29 + .../tests/unit/routes/yarn-queue/info-test.js | 29 + pom.xml | 4 +- 474 files changed, 13689 insertions(+), 6637 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --cc hadoop-project/pom.xml index eb47523,47e21d8..130460c --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@@ -1703,12 -1677,4 +1688,12 @@@ </build> </profile> </profiles> + + <repositories> + <repository> - <id>dynamodblocal</id> - <name>AWS DynamoDB Local Release Repository</name> - <url>http://dynamodb-local.s3-website-us-west-2.amazonaws.com/release</url> ++ <id>dynamodb-local-oregon</id> ++ <name>DynamoDB Local Release Repository</name> ++ <url>https://s3-us-west-2.amazonaws.com/dynamodb-local/release</url> + </repository> + </repositories> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/pom.xml index 73e2a39,0fdbc5d..ea93e96 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@@ -449,35 -355,6 +451,22 @@@ <artifactId>joda-time</artifactId> </dependency> <dependency> + <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-dynamodb</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.amazonaws</groupId> + <artifactId>DynamoDBLocal</artifactId> ++ <version>${aws-java-sdk.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </exclusion> + </exclusions> + </dependency> + <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-sts</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 3636186,414f951..6991a13 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@@ -302,114 -308,12 +313,122 @@@ public final class Constants @InterfaceAudience.Private public static final int MAX_MULTIPART_COUNT = 10000; + /** + * Classname of the S3A-specific output committer factory. This + * is what must be declared when attempting to use + */ + @InterfaceStability.Unstable + public static final String S3A_OUTPUT_COMMITTER_FACTORY = + "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory"; + + /* Constants. */ + public static final String S3_METADATA_STORE_IMPL = + "fs.s3a.metadatastore.impl"; + + /** Minimum period of time (in milliseconds) to keep metadata (may only be + * applied when a prune command is manually run). + */ + @InterfaceStability.Unstable + public static final String S3GUARD_CLI_PRUNE_AGE = + "fs.s3a.s3guard.cli.prune.age"; + + /** + * The endpoint of the DynamoDB service. + * + * This config has no default value. If the user does not set this, the AWS + * SDK will find the endpoint automatically by the Region. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_ENDPOINT_KEY = + "fs.s3a.s3guard.ddb.endpoint"; + /** + * The DynamoDB table name to use. + * + * This config has no default value. If the user does not set this, the + * S3Guard implementation will use the respective S3 bucket name. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_NAME_KEY = + "fs.s3a.s3guard.ddb.table"; + + /** + * Whether to create the DynamoDB table if the table does not exist. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CREATE_KEY = + "fs.s3a.s3guard.ddb.table.create"; + + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY = + "fs.s3a.s3guard.ddb.table.capacity.read"; + public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 500; + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY = + "fs.s3a.s3guard.ddb.table.capacity.write"; + public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 100; + + /** + * The maximum put or delete requests per BatchWriteItem request. + * + * Refer to Amazon API reference for this limit. + */ + public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25; + + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_MAX_RETRIES = + "fs.s3a.s3guard.ddb.max.retries"; + /** + * Max retries on batched DynamoDB operations before giving up and + * throwing an IOException. Default is {@value}. See core-default.xml for + * more detail. + */ + public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT = 9; + + /** + * Period of time (in milliseconds) to sleep between batches of writes. + * Currently only applies to prune operations, as they are naturally a + * lower priority than other operations. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY = + "fs.s3a.s3guard.ddb.background.sleep"; + public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25; + + /** + * V1 committer. + */ + @InterfaceStability.Unstable + public static final String S3A_OUTPUT_COMMITTER_MRV1 = + "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1"; + + /** + * The default "Null" metadata store: {@value}. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_NULL + = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore"; + + /** + * Use Local memory for the metadata: {@value}. + * This is not coherent across processes and must be used for testing only. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_LOCAL + = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore"; + + /** + * Use DynamoDB for the metadata: {@value}. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_DYNAMO + = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore"; + + @InterfaceAudience.Private + public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C + .getMethod() +" is enabled and no encryption key is provided."; + + + @InterfaceAudience.Private + public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3 + .getMethod() +" is configured and an " + "encryption key is provided"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index b93e567,8b1a6d0..c3f7a87 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@@ -2073,9 -2070,8 +2187,10 @@@ public class S3AFileSystem extends File if (blockFactory != null) { sb.append(", blockFactory=").append(blockFactory); } + sb.append(", metastore=").append(metadataStore); + sb.append(", authoritative=").append(allowAuthoritative); - sb.append(", executor=").append(threadPoolExecutor); + sb.append(", boundedExecutor=").append(boundedThreadPool); + sb.append(", unboundedExecutor=").append(unboundedThreadPool); sb.append(", statistics {") .append(statistics) .append("}"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2965cd8b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
