Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8094c38e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8094c38e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8094c38e Branch: refs/heads/HDFS-12943 Commit: 8094c38efdbfd11519ce518636cb630ac50c5f93 Parents: bba8aa3 3bfd214 Author: Konstantin V Shvachko <[email protected]> Authored: Tue Oct 16 16:32:13 2018 -0700 Committer: Konstantin V Shvachko <[email protected]> Committed: Tue Oct 16 16:32:13 2018 -0700 ---------------------------------------------------------------------- README.txt | 2 + .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../src/main/conf/log4j.properties | 6 + .../KeyProviderDelegationTokenExtension.java | 71 +- .../crypto/key/KeyProviderTokenIssuer.java | 4 +- .../crypto/key/kms/KMSClientProvider.java | 220 +++++-- .../key/kms/LoadBalancingKMSClientProvider.java | 75 ++- .../java/org/apache/hadoop/fs/FileSystem.java | 75 +-- .../org/apache/hadoop/security/Credentials.java | 1 + .../web/DelegationTokenAuthenticatedURL.java | 25 +- .../security/token/DelegationTokenIssuer.java | 111 ++++ .../java/org/apache/hadoop/util/KMSUtil.java | 13 +- .../conf/TestConfigurationDeprecation.java | 1 + ...TestKeyProviderDelegationTokenExtension.java | 20 +- .../crypto/key/kms/TestKMSClientProvider.java | 138 ++++ .../kms/TestLoadBalancingKMSClientProvider.java | 63 +- .../apache/hadoop/fs/TestFilterFileSystem.java | 3 + .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 + .../hadoop/security/ssl/KeyStoreTestUtil.java | 21 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 349 ++++++++-- .../hadoop/hdds/scm/XceiverClientManager.java | 17 +- .../scm/client/ContainerOperationClient.java | 15 +- hadoop-hdds/common/pom.xml | 6 - .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../hadoop/hdds/scm/pipeline/Pipeline.java | 211 ++++++ .../hadoop/hdds/scm/pipeline/PipelineID.java | 80 +++ .../hadoop/hdds/scm/pipeline/package-info.java | 24 + .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../apache/hadoop/utils/MetadataKeyFilters.java | 3 +- .../hadoop/utils/MetadataStoreBuilder.java | 38 +- .../main/proto/DatanodeContainerProtocol.proto | 4 +- .../main/proto/ScmBlockLocationProtocol.proto | 1 - .../StorageContainerLocationProtocol.proto | 1 - .../apache/hadoop/utils/TestMetadataStore.java | 2 + .../hadoop/utils/TestRocksDBStoreMBean.java | 19 +- hadoop-hdds/container-service/pom.xml | 11 +- .../common/helpers/ContainerReport.java | 205 ------ .../common/helpers/KeyValueContainerReport.java | 117 ---- .../container/common/impl/HddsDispatcher.java | 63 +- .../container/common/interfaces/Container.java | 5 + .../container/common/utils/ContainerCache.java | 6 +- .../container/keyvalue/KeyValueContainer.java | 9 +- .../keyvalue/KeyValueContainerData.java | 16 + .../container/keyvalue/KeyValueHandler.java | 30 +- .../container/keyvalue/helpers/BlockUtils.java | 3 +- .../container/keyvalue/helpers/ChunkUtils.java | 11 +- .../keyvalue/impl/BlockManagerImpl.java | 43 +- .../keyvalue/impl/ChunkManagerImpl.java | 43 ++ .../container/ozoneimpl/ContainerReader.java | 7 + .../StorageContainerDatanodeProtocol.proto | 2 + .../hdds/scm/pipeline/PipelineFactory.java | 56 ++ .../hdds/scm/pipeline/PipelineManager.java | 58 ++ .../hdds/scm/pipeline/PipelineProvider.java | 35 + .../hdds/scm/pipeline/PipelineStateManager.java | 179 +++++ .../hdds/scm/pipeline/PipelineStateMap.java | 212 ++++++ .../scm/pipeline/RatisPipelineProvider.java | 135 ++++ .../hdds/scm/pipeline/SCMPipelineManager.java | 226 +++++++ .../scm/pipeline/SimplePipelineProvider.java | 80 +++ .../hadoop/hdds/scm/pipeline/package-info.java | 24 + .../scm/server/StorageContainerManager.java | 8 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 11 +- .../hadoop/hdfs/DistributedFileSystem.java | 14 +- .../org/apache/hadoop/hdfs/HdfsKMSUtil.java | 60 +- .../apache/hadoop/hdfs/util/LongBitFormat.java | 8 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 29 +- .../resolver/order/RandomResolver.java | 22 +- .../main/webapps/router/federationhealth.html | 1 - .../server/namenode/AclEntryStatusFormat.java | 109 ++-- .../server/namenode/FSImageFormatPBINode.java | 101 +-- .../server/namenode/FSImageFormatProtobuf.java | 26 +- .../namenode/INodeWithAdditionalFields.java | 36 +- .../server/namenode/NameNodeLayoutVersion.java | 3 +- .../server/namenode/SerialNumberManager.java | 152 ++++- .../hdfs/server/namenode/SerialNumberMap.java | 43 +- .../hdfs/server/namenode/XAttrFormat.java | 95 +-- .../hdfs/server/namenode/XAttrStorage.java | 11 - .../tools/offlineImageViewer/FSImageLoader.java | 18 +- .../offlineImageViewer/PBImageTextWriter.java | 3 +- .../offlineImageViewer/PBImageXmlWriter.java | 22 +- .../hadoop-hdfs/src/main/proto/fsimage.proto | 1 + .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 32 +- .../datanode/TestDataNodeVolumeFailure.java | 7 +- .../TestCommitBlockSynchronization.java | 6 +- .../org/apache/hadoop/mapreduce/MRConfig.java | 2 +- .../counters/FileSystemCounterGroup.java | 56 +- .../mapreduce/jobhistory/JobHistoryParser.java | 2 +- .../src/main/resources/mapred-default.xml | 2 +- .../split/TestJobSplitWriterWithEC.java | 128 ++++ .../hadoop/mapreduce/v2/hs/CompletedTask.java | 5 +- .../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +- .../apache/hadoop/ozone/client/BucketArgs.java | 2 +- .../apache/hadoop/ozone/client/ObjectStore.java | 9 + .../apache/hadoop/ozone/client/OzoneBucket.java | 2 +- .../ozone/client/io/ChunkGroupInputStream.java | 8 +- .../ozone/client/io/ChunkGroupOutputStream.java | 3 +- .../ozone/client/protocol/ClientProtocol.java | 10 +- .../hadoop/ozone/client/rest/RestClient.java | 9 +- .../hadoop/ozone/client/rpc/RpcClient.java | 10 +- hadoop-ozone/common/pom.xml | 6 - .../hadoop/hdds/protocol/StorageType.java | 64 ++ .../ozone/client/rest/response/BucketInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 7 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 9 +- .../ozone/om/helpers/OmKeyLocationInfo.java | 4 + .../ozone/om/protocol/OzoneManagerProtocol.java | 7 + ...neManagerProtocolClientSideTranslatorPB.java | 23 + .../hadoop/ozone/web/handlers/BucketArgs.java | 2 +- .../hadoop/ozone/web/response/BucketInfo.java | 2 +- .../src/main/proto/OzoneManagerProtocol.proto | 23 +- .../apache/hadoop/ozone/web/TestBucketInfo.java | 2 +- .../src/main/compose/ozone-hdfs/docker-config | 1 + .../dist/src/main/compose/ozone/docker-config | 1 + .../dist/src/main/compose/ozonefs/docker-config | 2 + .../src/main/compose/ozoneperf/docker-config | 3 + .../dist/src/main/compose/ozones3/docker-config | 6 + .../src/main/compose/ozonescripts/docker-config | 2 + .../dist/src/main/compose/ozonescripts/start.sh | 4 +- .../dist/src/main/smoketest/commonlib.robot | 7 + .../src/main/smoketest/ozonefs/ozonefs.robot | 12 +- .../dist/src/main/smoketest/s3/README.md | 27 + .../dist/src/main/smoketest/s3/__init__.robot | 21 + .../dist/src/main/smoketest/s3/awscli.robot | 63 -- .../dist/src/main/smoketest/s3/awss3.robot | 47 ++ .../dist/src/main/smoketest/s3/buckethead.robot | 34 + .../dist/src/main/smoketest/s3/bucketlist.robot | 32 + .../src/main/smoketest/s3/commonawslib.robot | 67 ++ .../src/main/smoketest/s3/objectdelete.robot | 72 +++ .../src/main/smoketest/s3/objectputget.robot | 42 ++ hadoop-ozone/dist/src/main/smoketest/test.sh | 26 +- hadoop-ozone/docs/content/Dozone.md | 4 +- hadoop-ozone/docs/content/JavaApi.md | 2 +- hadoop-ozone/docs/content/OzoneManager.md | 2 +- hadoop-ozone/docs/content/RealCluster.md | 10 +- hadoop-ozone/docs/content/Settings.md | 2 +- .../scm/pipeline/TestPipelineStateManager.java | 246 +++++++ .../scm/pipeline/TestRatisPipelineProvider.java | 104 +++ .../pipeline/TestSimplePipelineProvider.java | 102 +++ .../hadoop/ozone/MiniOzoneClusterImpl.java | 2 + .../TestContainerStateMachineIdempotency.java | 121 ++++ .../TestStorageContainerManagerHelper.java | 2 +- .../ozone/client/rest/TestOzoneRestClient.java | 2 +- .../hadoop/ozone/client/rpc/TestBCSID.java | 147 +++++ .../rpc/TestContainerStateMachineFailures.java | 185 ++++++ .../ozone/client/rpc/TestOzoneRpcClient.java | 30 +- .../common/impl/TestContainerPersistence.java | 17 +- .../om/TestMultipleContainerReadWrite.java | 2 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 2 +- .../org/apache/hadoop/ozone/om/TestOmInit.java | 103 +++ .../hadoop/ozone/om/TestOzoneManager.java | 2 +- .../hadoop/ozone/ozShell/TestOzoneShell.java | 22 + .../ozone/scm/TestContainerSmallFile.java | 9 +- .../TestGetCommittedBlockLengthAndPutKey.java | 12 +- .../ozone/scm/TestXceiverClientManager.java | 30 +- .../ozone/scm/TestXceiverClientMetrics.java | 3 +- .../web/handlers/BucketProcessTemplate.java | 2 +- .../hadoop/ozone/om/BucketManagerImpl.java | 2 +- .../apache/hadoop/ozone/om/OzoneManager.java | 23 +- .../apache/hadoop/ozone/om/S3BucketManager.java | 7 + .../hadoop/ozone/om/S3BucketManagerImpl.java | 27 +- ...neManagerProtocolServerSideTranslatorPB.java | 18 + .../web/ozShell/volume/CreateVolumeHandler.java | 7 +- .../hadoop/ozone/om/TestBucketManagerImpl.java | 2 +- .../hadoop/ozone/om/TestChunkStreams.java | 2 +- .../hadoop/ozone/om/TestS3BucketManager.java | 18 + hadoop-ozone/ozonefs/pom.xml | 9 + .../services/org.apache.hadoop.fs.FileSystem | 16 + .../fs/ozone/TestOzoneFileInterfaces.java | 11 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 39 ++ .../apache/hadoop/ozone/s3/EndpointBase.java | 87 --- .../hadoop/ozone/s3/bucket/DeleteBucket.java | 68 -- .../hadoop/ozone/s3/bucket/HeadBucket.java | 64 -- .../hadoop/ozone/s3/bucket/PutBucket.java | 44 -- .../hadoop/ozone/s3/bucket/package-info.java | 30 - .../ozone/s3/commontypes/BucketMetadata.java | 53 ++ .../ozone/s3/commontypes/IsoDateAdapter.java | 2 +- .../ozone/s3/endpoint/BucketEndpoint.java | 199 ++++++ .../hadoop/ozone/s3/endpoint/EndpointBase.java | 213 ++++++ .../ozone/s3/endpoint/ListBucketResponse.java | 55 ++ .../ozone/s3/endpoint/ListObjectResponse.java | 158 +++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 222 +++++++ .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 82 +++ .../hadoop/ozone/s3/endpoint/package-info.java | 30 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 12 +- .../ozone/s3/header/AuthorizationHeaderV2.java | 100 +++ .../ozone/s3/header/AuthorizationHeaderV4.java | 147 +++++ .../hadoop/ozone/s3/header/Credential.java | 94 +++ .../hadoop/ozone/s3/header/package-info.java | 22 + .../hadoop/ozone/s3/object/DeleteObject.java | 51 -- .../hadoop/ozone/s3/object/HeadObject.java | 74 --- .../hadoop/ozone/s3/object/ListObject.java | 119 ---- .../ozone/s3/object/ListObjectResponse.java | 147 ----- .../hadoop/ozone/s3/object/PutObject.java | 61 -- .../hadoop/ozone/s3/object/package-info.java | 29 - .../hadoop/ozone/client/ObjectStoreStub.java | 58 ++ .../hadoop/ozone/client/OzoneBucketStub.java | 2 +- .../hadoop/ozone/client/OzoneVolumeStub.java | 22 +- .../ozone/s3/bucket/TestBucketResponse.java | 40 -- .../ozone/s3/bucket/TestDeleteBucket.java | 106 --- .../hadoop/ozone/s3/bucket/TestGetBucket.java | 114 ---- .../hadoop/ozone/s3/bucket/TestHeadBucket.java | 92 --- .../hadoop/ozone/s3/bucket/package-info.java | 21 - .../ozone/s3/endpoint/TestBucketDelete.java | 100 +++ .../hadoop/ozone/s3/endpoint/TestBucketGet.java | 115 ++++ .../ozone/s3/endpoint/TestBucketHead.java | 71 ++ .../ozone/s3/endpoint/TestBucketResponse.java | 38 ++ .../ozone/s3/endpoint/TestObjectDelete.java | 60 ++ .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 80 +++ .../ozone/s3/endpoint/TestObjectHead.java | 96 +++ .../hadoop/ozone/s3/endpoint/TestPutObject.java | 91 +++ .../hadoop/ozone/s3/endpoint/TestRootList.java | 79 +++ .../hadoop/ozone/s3/endpoint/package-info.java | 21 + .../s3/header/TestAuthorizationHeaderV2.java | 88 +++ .../s3/header/TestAuthorizationHeaderV4.java | 81 +++ .../ozone/s3/object/TestDeleteObject.java | 56 -- .../hadoop/ozone/s3/object/TestHeadObject.java | 95 --- .../hadoop/ozone/s3/object/package-info.java | 21 - hadoop-project/pom.xml | 2 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 +- .../hadoop/fs/s3a/WriteOperationHelper.java | 4 +- .../fs/azurebfs/utils/SSLSocketFactoryEx.java | 3 + .../hadoop/tools/CopyListingFileStatus.java | 3 +- .../hadoop/tools/TestCopyListingFileStatus.java | 10 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 29 + .../hadoop/yarn/api/ApplicationConstants.java | 22 + .../yarn/api/records/ResourceInformation.java | 68 +- .../hadoop/yarn/conf/YarnConfiguration.java | 20 + .../yarn/util/resource/ResourceUtils.java | 9 +- .../src/main/proto/yarn_protos.proto | 2 + .../yarn/conf/TestResourceInformation.java | 44 ++ .../yarn/service/client/ApiServiceClient.java | 19 +- .../client/TestSecureApiServiceClient.java | 113 +++- .../yarn/service/UpgradeComponentsFinder.java | 2 +- .../yarn/service/api/records/ConfigFile.java | 3 +- .../yarn/service/utils/ServiceApiUtil.java | 5 +- .../TestDefaultUpgradeComponentsFinder.java | 40 +- .../yarn/service/utils/TestServiceApiUtil.java | 8 + .../installation/install.conf | 74 +++ .../installation/install.sh | 116 ++++ .../package/calico/calico-node.service | 50 ++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 + .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 + .../installation/scripts/calico.sh | 224 +++++++ .../installation/scripts/docker.sh | 166 +++++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 ++++++ .../installation/scripts/etcd.sh | 152 +++++ .../installation/scripts/hadoop.sh | 117 ++++ .../installation/scripts/menu.sh | 444 +++++++++++++ .../installation/scripts/nvidia-docker.sh | 99 +++ .../installation/scripts/nvidia.sh | 120 ++++ .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 ++++ .../client/cli/param/RunJobParameters.java | 6 + .../src/site/markdown/HowToInstall.md | 36 ++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +----- .../src/site/markdown/InstallationScriptCN.md | 242 +++++++ .../src/site/markdown/InstallationScriptEN.md | 250 +++++++ .../src/site/markdown/TestAndTroubleshooting.md | 165 +++++ .../resources/images/submarine-installer.gif | Bin 0 -> 546547 bytes .../client/cli/TestRunJobCliParsing.java | 39 ++ .../yarn/api/records/impl/pb/ProtoUtils.java | 28 + .../api/records/impl/pb/ResourcePBImpl.java | 23 + .../src/main/resources/yarn-default.xml | 18 + .../hadoop/yarn/api/TestResourcePBImpl.java | 128 +++- .../resourcetypes/ResourceTypesTestHelper.java | 22 + .../resource-types/resource-types-5.xml | 53 ++ .../yarn/server/security/AMSecretKeys.java | 45 ++ .../nodemanager/DefaultContainerExecutor.java | 14 + .../nodemanager/LinuxContainerExecutor.java | 4 + .../nodemanager/LocalDirsHandlerService.java | 5 +- .../launcher/ContainerLaunch.java | 50 ++ .../launcher/ContainerRelaunch.java | 23 + .../runtime/DefaultLinuxContainerRuntime.java | 14 +- .../runtime/DockerLinuxContainerRuntime.java | 36 +- .../runtime/LinuxContainerRuntimeConstants.java | 4 + .../executor/ContainerStartContext.java | 24 + .../impl/container-executor.c | 121 +++- .../impl/container-executor.h | 17 +- .../main/native/container-executor/impl/main.c | 35 +- .../main/native/container-executor/impl/util.h | 6 +- .../test/test-container-executor.c | 228 ++++--- .../TestDefaultContainerExecutor.java | 157 +++++ .../TestLinuxContainerExecutorWithMocks.java | 115 ++-- .../launcher/TestContainerLaunch.java | 149 +++++ .../launcher/TestContainerRelaunch.java | 32 +- .../runtime/TestDockerContainerRuntime.java | 225 +++---- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 11 + .../server/resourcemanager/ResourceManager.java | 9 + .../resourcemanager/amlauncher/AMLauncher.java | 29 + .../resourcemanager/scheduler/QueueMetrics.java | 130 +++- .../QueueMetricsForCustomResources.java | 158 +++++ .../scheduler/capacity/CapacityScheduler.java | 5 +- .../security/ProxyCAManager.java | 68 ++ .../TestApplicationMasterLauncher.java | 86 ++- .../resourcemanager/scheduler/QueueInfo.java | 90 +++ .../scheduler/QueueMetricsTestData.java | 105 +++ .../scheduler/ResourceMetricsChecker.java | 88 ++- .../scheduler/TestQueueMetrics.java | 250 +++---- .../TestQueueMetricsForCustomResources.java | 645 +++++++++++++++++++ .../security/TestProxyCAManager.java | 51 ++ .../hadoop/yarn/server/TestDiskFailures.java | 38 +- .../hadoop/yarn/server/webproxy/ProxyCA.java | 408 ++++++++++++ .../yarn/server/webproxy/WebAppProxy.java | 1 + .../server/webproxy/WebAppProxyServlet.java | 89 ++- .../yarn/server/webproxy/TestProxyCA.java | 518 +++++++++++++++ .../server/webproxy/TestWebAppProxyServlet.java | 58 +- .../src/site/markdown/YarnUI2.md | 2 +- 315 files changed, 14607 insertions(+), 3305 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8094c38e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
