This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-1880-Decom in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ec70207838d5b29fa0b534b13c103865e50a35e8 Merge: fd5e877 6171a41 Author: Márton Elek <e...@apache.org> AuthorDate: Fri Oct 4 14:17:38 2019 +0200 Merge remote-tracking branch 'origin/trunk' into HDDS-1880-Decom BUILDING.txt | 31 - dev-support/docker/Dockerfile | 19 +- .../hadoop-cos/dev-support/findbugs-exclude.xml | 18 + hadoop-cloud-storage-project/hadoop-cos/pom.xml | 140 ++++ .../site/markdown/cloud-storage/index.md | 367 ++++++++++ .../hadoop-cos/site/resources/css/site.css | 29 + .../java/org/apache/hadoop/fs/cosn/BufferPool.java | 245 +++++++ .../hadoop/fs/cosn/ByteBufferInputStream.java | 89 +++ .../hadoop/fs/cosn/ByteBufferOutputStream.java | 74 ++ .../apache/hadoop/fs/cosn/ByteBufferWrapper.java | 103 +++ .../java/org/apache/hadoop/fs/cosn/Constants.java | 35 +- .../main/java/org/apache/hadoop/fs/cosn/CosN.java | 31 +- .../org/apache/hadoop/fs/cosn/CosNConfigKeys.java | 86 +++ .../apache/hadoop/fs/cosn/CosNCopyFileContext.java | 66 ++ .../apache/hadoop/fs/cosn/CosNCopyFileTask.java | 68 ++ .../apache/hadoop/fs/cosn/CosNFileReadTask.java | 125 ++++ .../org/apache/hadoop/fs/cosn/CosNFileSystem.java | 814 +++++++++++++++++++++ .../org/apache/hadoop/fs/cosn/CosNInputStream.java | 365 +++++++++ .../apache/hadoop/fs/cosn/CosNOutputStream.java | 284 +++++++ .../java/org/apache/hadoop/fs/cosn/CosNUtils.java | 167 +++++ .../hadoop/fs/cosn/CosNativeFileSystemStore.java | 768 +++++++++++++++++++ .../org/apache/hadoop/fs/cosn/FileMetadata.java | 68 ++ .../hadoop/fs/cosn/NativeFileSystemStore.java | 99 +++ .../org/apache/hadoop/fs/cosn/PartialListing.java | 64 ++ .../main/java/org/apache/hadoop/fs/cosn/Unit.java | 27 +- .../fs/cosn/auth/COSCredentialProviderList.java | 139 ++++ .../EnvironmentVariableCredentialProvider.java | 55 ++ .../fs/cosn/auth/NoAuthWithCOSException.java | 32 +- .../fs/cosn/auth/SimpleCredentialProvider.java | 54 ++ .../apache/hadoop/fs/cosn/auth/package-info.java | 19 +- .../org/apache/hadoop/fs/cosn/package-info.java | 19 +- .../apache/hadoop/fs/cosn/CosNTestConfigKey.java | 30 +- .../org/apache/hadoop/fs/cosn/CosNTestUtils.java | 78 ++ .../apache/hadoop/fs/cosn/TestCosNInputStream.java | 167 +++++ .../hadoop/fs/cosn/TestCosNOutputStream.java | 87 +++ .../hadoop/fs/cosn/contract/CosNContract.java | 36 +- .../fs/cosn/contract/TestCosNContractCreate.java | 26 +- .../fs/cosn/contract/TestCosNContractDelete.java | 26 +- .../fs/cosn/contract/TestCosNContractDistCp.java | 54 ++ .../contract/TestCosNContractGetFileStatus.java | 27 +- .../fs/cosn/contract/TestCosNContractMkdir.java | 26 +- .../fs/cosn/contract/TestCosNContractOpen.java | 26 +- .../fs/cosn/contract/TestCosNContractRename.java | 26 +- .../fs/cosn/contract/TestCosNContractRootDir.java | 27 +- .../fs/cosn/contract/TestCosNContractSeek.java | 26 +- .../hadoop/fs/cosn/contract/package-info.java | 19 +- .../src/test/resources/contract/cosn.xml | 120 +++ .../hadoop-cos/src/test/resources/core-site.xml | 107 +++ .../hadoop-cos/src/test/resources/log4j.properties | 18 + hadoop-cloud-storage-project/pom.xml | 1 + .../apache/hadoop/crypto/CryptoInputStream.java | 67 +- .../org/apache/hadoop/fs/AbstractFileSystem.java | 16 +- .../hadoop/fs/ByteBufferPositionedReadable.java | 24 + .../org/apache/hadoop/fs/ChecksumFileSystem.java | 22 + .../apache/hadoop/fs/CommonPathCapabilities.java | 126 ++++ .../org/apache/hadoop/fs/DelegateToFileSystem.java | 7 + .../org/apache/hadoop/fs/FSDataInputStream.java | 23 +- .../java/org/apache/hadoop/fs/FileContext.java | 23 +- .../main/java/org/apache/hadoop/fs/FileSystem.java | 30 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 7 + .../main/java/org/apache/hadoop/fs/FilterFs.java | 5 + .../main/java/org/apache/hadoop/fs/Globber.java | 208 +++++- .../java/org/apache/hadoop/fs/HarFileSystem.java | 19 +- .../org/apache/hadoop/fs/PathCapabilities.java | 61 ++ .../org/apache/hadoop/fs/RawLocalFileSystem.java | 19 + .../hadoop/fs/http/AbstractHttpFileSystem.java | 18 + .../apache/hadoop/fs/impl/FsLinkResolution.java | 98 +++ .../hadoop/fs/impl/PathCapabilitiesSupport.java | 40 +- .../java/org/apache/hadoop/fs/shell/Mkdir.java | 4 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 32 + .../apache/hadoop/util/NodeHealthScriptRunner.java | 1 + .../src/site/markdown/DeprecatedProperties.md | 4 + .../src/site/markdown/filesystem/filesystem.md | 5 +- .../src/site/markdown/filesystem/index.md | 1 + .../site/markdown/filesystem/pathcapabilities.md | 158 ++++ .../site/markdown/release/3.1.3/CHANGES.3.1.3.md | 336 +++++++++ .../markdown/release/3.1.3/RELEASENOTES.3.1.3.md | 59 ++ .../hadoop/crypto/CryptoStreamsTestBase.java | 115 ++- .../apache/hadoop/crypto/TestCryptoStreams.java | 24 + .../hadoop/crypto/TestCryptoStreamsForLocalFS.java | 5 + .../hadoop/crypto/TestCryptoStreamsNormal.java | 5 + .../fs/contract/AbstractContractAppendTest.java | 10 + .../fs/contract/AbstractContractConcatTest.java | 7 + .../hadoop/fs/contract/ContractTestUtils.java | 50 +- .../org/apache/hadoop/test/LambdaTestUtils.java | 3 + .../hadoop/util/TestNodeHealthScriptRunner.java | 9 + hadoop-hdds/common/pom.xml | 4 +- .../function/FunctionWithServiceException.java | 29 +- .../apache/hadoop/hdds/function/package-info.java | 15 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 104 ++- .../SCMSecurityProtocolServerSideTranslatorPB.java | 132 ---- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 + ...inerLocationProtocolClientSideTranslatorPB.java | 411 +++++------ .../hadoop/hdds/utils/db/DBStoreBuilder.java | 24 +- .../hadoop/hdds/utils/db/RocksDBConfiguration.java | 62 ++ .../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +- .../hadoop/hdds/utils/db/cache/TableCache.java | 9 + .../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 7 + .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../org/apache/hadoop/ozone/lock/ActiveLock.java | 63 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 166 ++++- ...inerLocationProtocolServerSideTranslatorPB.java | 389 ---------- .../src/main/proto/SCMSecurityProtocol.proto | 96 ++- .../src/main/proto/ScmBlockLocationProtocol.proto | 2 +- .../proto/StorageContainerLocationProtocol.proto | 185 +++-- .../common/src/main/resources/ozone-default.xml | 8 + .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 16 +- .../apache/hadoop/ozone/lock/TestLockManager.java | 145 +++- .../hadoop/hdds/conf/ConfigFileGenerator.java | 5 +- hadoop-hdds/container-service/pom.xml | 9 +- .../server/ratis/ContainerStateMachine.java | 20 +- .../transport/server/ratis/XceiverServerRatis.java | 8 +- .../replication/GrpcReplicationClient.java | 6 + ...inerDatanodeProtocolClientSideTranslatorPB.java | 60 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 115 +-- .../proto/StorageContainerDatanodeProtocol.proto | 58 +- .../ozone/container/common/SCMTestUtils.java | 4 +- .../server/OzoneProtocolMessageDispatcher.java | 88 +++ .../apache/hadoop/hdds/server/ProfileServlet.java | 10 +- .../hadoop/hdds/server/PrometheusMetricsSink.java | 16 +- .../hdds/server/TestPrometheusMetricsSink.java | 77 +- hadoop-hdds/pom.xml | 17 +- hadoop-hdds/server-scm/pom.xml | 4 +- .../container/AbstractContainerReportHandler.java | 8 + .../hdds/scm/container/ContainerManager.java | 8 + .../hdds/scm/container/ContainerReportHandler.java | 2 + .../IncrementalContainerReportHandler.java | 10 + .../hdds/scm/container/SCMContainerManager.java | 48 +- .../metrics/SCMContainerManagerMetrics.java | 144 ++++ .../hdds/scm/container/metrics/package-info.java | 22 + .../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 51 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 186 +++++ ...lockLocationProtocolServerSideTranslatorPB.java | 56 +- ...inerLocationProtocolServerSideTranslatorPB.java | 393 ++++++++++ .../hadoop/hdds/scm/protocol/package-info.java | 22 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 26 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 9 +- .../hdds/scm/server/SCMClientProtocolServer.java | 15 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 102 ++- .../hdds/scm/server/SCMSecurityProtocolServer.java | 27 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 36 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 67 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +- .../scm/server/TestSCMBlockProtocolServer.java | 3 +- .../testutils/ReplicationNodeManagerMock.java | 5 +- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 22 +- .../hdds/scm/cli/container/CloseSubcommand.java | 7 +- ...CloseSubcommand.java => ContainerCommands.java} | 37 +- .../hdds/scm/cli/container/CreateSubcommand.java | 5 +- .../hdds/scm/cli/container/DeleteSubcommand.java | 7 +- .../hdds/scm/cli/container/InfoSubcommand.java | 5 +- .../hdds/scm/cli/container/ListSubcommand.java | 5 +- .../cli/pipeline/ActivatePipelineSubcommand.java | 11 +- .../scm/cli/pipeline/ClosePipelineSubcommand.java | 11 +- .../cli/pipeline/DeactivatePipelineSubcommand.java | 11 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +- .../PipelineCommands.java} | 38 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 14 + .../apache/hadoop/hdfs/DistributedFileSystem.java | 22 + .../hadoop/hdfs/client/DfsPathCapabilities.java | 62 ++ .../org/apache/hadoop/hdfs/protocol/Block.java | 141 +++- .../namenode/ha/ObserverReadProxyProvider.java | 26 +- .../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 30 + .../hadoop/hdfs/TestDefaultNameNodePort.java | 38 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 30 + .../fs/http/server/HttpFSAuthenticationFilter.java | 16 +- .../fs/http/server/HttpFSServerWebServer.java | 22 + .../service/hadoop/FileSystemAccessService.java | 21 +- .../src/main/resources/httpfs-default.xml | 20 +- .../main/native/libhdfs-tests/test_libhdfs_ops.c | 43 +- .../src/main/native/libhdfs/hdfs.c | 117 +++ .../src/main/native/libhdfs/include/hdfs/hdfs.h | 21 +- .../src/main/native/libhdfspp/tests/hdfs_shim.c | 6 + .../libhdfspp/tests/libhdfs_wrapper_defines.h | 1 + .../libhdfspp/tests/libhdfs_wrapper_undefs.h | 1 + .../libhdfspp/tests/libhdfspp_wrapper_defines.h | 1 + hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 48 ++ .../src/main/webapps/router/federationhealth.html | 10 +- .../src/main/webapps/router/federationhealth.js | 7 +- .../src/main/webapps/static/rbf.css | 26 +- .../fs/contract/router/RouterHDFSContract.java | 17 +- .../fs/contract/router/SecurityConfUtil.java | 15 +- .../router/TestRouterHDFSContractAppendSecure.java | 4 +- .../router/TestRouterHDFSContractConcatSecure.java | 5 +- .../router/TestRouterHDFSContractCreateSecure.java | 5 +- .../TestRouterHDFSContractDelegationToken.java | 2 +- .../router/TestRouterHDFSContractDeleteSecure.java | 4 +- .../TestRouterHDFSContractGetFileStatusSecure.java | 5 +- .../router/TestRouterHDFSContractMkdirSecure.java | 5 +- .../router/TestRouterHDFSContractOpenSecure.java | 5 +- .../router/TestRouterHDFSContractRenameSecure.java | 5 +- .../TestRouterHDFSContractRootDirectorySecure.java | 5 +- .../router/TestRouterHDFSContractSeekSecure.java | 5 +- .../TestRouterHDFSContractSetTimesSecure.java | 5 +- .../server/federation/router/TestRouterQuota.java | 2 +- .../security/TestRouterHttpDelegationToken.java | 1 + .../dev-support/jdiff/Apache_Hadoop_HDFS_3.1.3.xml | 676 +++++++++++++++++ .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 +- .../hdfs/qjournal/client/IPCLoggerChannel.java | 14 +- .../hadoop/hdfs/qjournal/client/QuorumCall.java | 18 + .../hdfs/qjournal/client/QuorumJournalManager.java | 2 + .../token/block/BlockTokenSecretManager.java | 31 +- .../hdfs/server/blockmanagement/BlockInfo.java | 6 + .../blockmanagement/BlockInfoContiguous.java | 14 + .../server/blockmanagement/BlockInfoStriped.java | 9 + .../hdfs/server/blockmanagement/BlockManager.java | 94 ++- .../blockmanagement/BlockPlacementStatus.java | 8 + .../BlockPlacementStatusDefault.java | 8 + .../BlockPlacementStatusWithNodeGroup.java | 11 + .../BlockPlacementStatusWithUpgradeDomain.java | 22 +- .../blockmanagement/BlockReportLeaseManager.java | 8 +- .../hdfs/server/namenode/INodeDirectory.java | 6 + .../hadoop/hdfs/server/namenode/INodeFile.java | 3 + .../hdfs/server/namenode/NameNodeRpcServer.java | 14 +- .../namenode/snapshot/AbstractINodeDiffList.java | 4 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- .../src/main/resources/hdfs-default.xml | 19 + .../src/main/webapps/hdfs/dfshealth.html | 4 +- .../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 +- .../src/site/markdown/ObserverNameNode.md | 14 + .../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 17 +- .../hadoop/hdfs/TestBlockMissingException.java | 2 - .../apache/hadoop/hdfs/TestByteBufferPread.java | 27 +- .../java/org/apache/hadoop/hdfs/TestClose.java | 1 - .../hdfs/TestStateAlignmentContextWithHA.java | 1 + .../hadoop/hdfs/qjournal/MiniJournalCluster.java | 6 +- .../qjournal/client/TestQuorumJournalManager.java | 34 +- .../hdfs/security/token/block/TestBlockToken.java | 23 + .../blockmanagement/BlockManagerTestUtil.java | 30 +- .../hdfs/server/blockmanagement/TestBlockInfo.java | 36 + .../server/blockmanagement/TestBlockManager.java | 58 ++ .../TestBlockPlacementStatusDefault.java | 57 ++ .../TestBlockPlacementStatusWithUpgradeDomain.java | 59 +- .../TestBlocksWithNotEnoughRacks.java | 111 ++- .../namenode/ha/TestConsistentReadsObserver.java | 2 + .../ha/TestFailoverWithBlockTokensEnabled.java | 36 +- .../server/namenode/ha/TestMultiObserverNode.java | 2 + .../hdfs/server/namenode/ha/TestObserverNode.java | 2 + .../namenode/snapshot/TestRenameWithSnapshots.java | 15 +- .../namenode/snapshot/TestSnapshotDeletion.java | 2 +- .../hadoop-mapreduce-client-common/pom.xml | 34 +- .../src/main/proto/HSAdminRefreshProtocol.proto | 3 +- .../src/main/proto/MRClientProtocol.proto | 1 + .../src/main/proto/mr_protos.proto | 1 + .../src/main/proto/mr_service_protos.proto | 1 + .../org/apache/hadoop/mapred/FileInputFormat.java | 5 +- .../hadoop/mapred/InvalidInputException.java | 4 + .../hadoop/mapred/LocatedFileStatusFetcher.java | 66 +- .../mapreduce/lib/input/FileInputFormat.java | 6 +- .../mapreduce/lib/input/InvalidInputException.java | 4 + .../hadoop-mapreduce-client-shuffle/pom.xml | 27 +- .../src/main/proto/ShuffleHandlerRecovery.proto | 1 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +- hadoop-ozone/common/pom.xml | 4 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 57 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +- .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 52 ++ ...lumeListCodec.java => UserVolumeInfoCodec.java} | 13 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 100 ++- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 91 +++ .../hadoop/ozone/security/GDPRSymmetricKey.java | 8 +- .../OzoneDelegationTokenSecretManager.java | 5 +- .../hadoop/ozone/security/OzoneSecretManager.java | 7 +- .../src/main/proto/OzoneManagerProtocol.proto | 18 +- .../ozone/security/TestGDPRSymmetricKey.java | 4 +- hadoop-ozone/csi/pom.xml | 4 +- .../dev-support/checks/_mvn_unit_report.sh | 16 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 12 +- .../compose/ozonesecure-mr/docker-compose.yaml | 39 +- .../src/main/compose/ozonesecure-mr/docker-config | 31 +- .../{test-single.sh => ozonesecure-mr/test.sh} | 37 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 4 +- hadoop-ozone/dist/src/main/compose/test-single.sh | 2 + hadoop-ozone/dist/src/main/compose/testlib.sh | 27 +- .../dist/src/main/smoketest/gdpr/gdpr.robot | 89 +++ .../smoketest/{kinit.robot => kinit-hadoop.robot} | 2 +- hadoop-ozone/dist/src/main/smoketest/kinit.robot | 5 +- .../dist/src/main/smoketest/mapreduce.robot | 2 +- hadoop-ozone/insight/pom.xml | 9 +- .../ozone/insight/BaseInsightSubCommand.java | 7 +- .../scm/ScmProtocolBlockLocationInsight.java | 6 +- ...va => ScmProtocolContainerLocationInsight.java} | 18 +- ...nsight.java => ScmProtocolDatanodeInsight.java} | 27 +- ...nsight.java => ScmProtocolSecurityInsight.java} | 18 +- .../metrics/TestSCMContainerManagerMetrics.java | 167 +++++ .../scm/pipeline/TestRatisPipelineProvider.java | 26 +- .../rpc/TestContainerStateMachineFailures.java | 65 ++ .../client/rpc/TestOzoneRpcClientAbstract.java | 76 +- .../ozone/om/TestOzoneManagerConfiguration.java | 9 +- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 97 +++ hadoop-ozone/ozone-manager/pom.xml | 5 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 40 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 113 +-- .../org/apache/hadoop/ozone/om/OzoneManager.java | 232 +----- .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 35 +- .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 306 ++++++++ .../hadoop/ozone/om/{ => ha}/OMNodeDetails.java | 2 +- .../apache/hadoop/ozone/om/ha/package-info.java | 14 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +- .../request/s3/bucket/S3BucketCreateRequest.java | 8 +- .../multipart/S3MultipartUploadAbortRequest.java | 5 +- .../S3MultipartUploadCommitPartRequest.java | 4 +- .../om/request/volume/OMVolumeCreateRequest.java | 12 +- .../om/request/volume/OMVolumeDeleteRequest.java | 5 +- .../ozone/om/request/volume/OMVolumeRequest.java | 32 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 10 +- .../om/response/bucket/OMBucketCreateResponse.java | 8 +- .../om/response/bucket/OMBucketDeleteResponse.java | 4 +- .../bucket/OMBucketSetPropertyResponse.java | 7 +- .../response/file/OMDirectoryCreateResponse.java | 3 +- .../om/response/file/OMFileCreateResponse.java | 3 +- .../om/response/key/OMAllocateBlockResponse.java | 6 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 7 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 3 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 32 +- .../ozone/om/response/key/OMKeyPurgeResponse.java | 4 +- .../ozone/om/response/key/OMKeyRenameResponse.java | 6 +- .../multipart/S3MultipartUploadAbortResponse.java | 21 +- .../S3MultipartUploadCommitPartResponse.java | 45 +- .../S3MultipartUploadCompleteResponse.java | 3 +- .../om/response/volume/OMVolumeAclOpResponse.java | 3 +- .../om/response/volume/OMVolumeCreateResponse.java | 13 +- .../om/response/volume/OMVolumeDeleteResponse.java | 10 +- .../response/volume/OMVolumeSetOwnerResponse.java | 12 +- .../response/volume/OMVolumeSetQuotaResponse.java | 4 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 41 +- .../org/apache/hadoop/ozone/web/ozShell/Shell.java | 3 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 21 +- .../om/ratis/TestOzoneManagerRatisServer.java | 2 +- .../ozone/om/request/TestOMRequestUtils.java | 25 +- .../hadoop/ozone/om/request/package-info.java} | 18 +- .../request/volume/TestOMVolumeCreateRequest.java | 16 +- .../volume/TestOMVolumeSetOwnerRequest.java | 4 +- .../ozone/om/response/TestOMResponseUtils.java | 8 +- .../om/response/key/TestOMKeyDeleteResponse.java | 20 +- .../hadoop/ozone/om/response/package-info.java} | 18 +- .../s3/multipart/TestS3MultipartResponse.java | 3 +- .../TestS3MultipartUploadAbortResponse.java | 19 +- .../volume/TestOMVolumeCreateResponse.java | 7 +- .../volume/TestOMVolumeDeleteResponse.java | 11 +- .../volume/TestOMVolumeSetOwnerResponse.java | 17 +- .../ozone/om/response/volume/package-info.java | 22 +- .../TestOzoneDelegationTokenSecretManager.java | 29 +- hadoop-ozone/ozonefs-lib-current/pom.xml | 4 +- hadoop-ozone/ozonefs-lib-legacy/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 9 +- hadoop-ozone/pom.xml | 14 +- hadoop-ozone/recon/pom.xml | 4 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 35 + .../recon/recovery/ReconOmMetadataManagerImpl.java | 21 +- .../spi/impl/ContainerDBServiceProviderImpl.java | 28 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 33 +- .../recon/spi/impl/ReconContainerDBProvider.java | 32 +- .../ozone/recon/AbstractOMMetadataManagerTest.java | 2 +- .../apache/hadoop/ozone/recon/TestReconUtils.java | 31 + .../recovery/TestReconOmMetadataManagerImpl.java | 133 ++-- .../impl/TestOzoneManagerServiceProviderImpl.java | 29 +- .../spi/impl/TestReconContainerDBProvider.java | 13 - hadoop-ozone/s3gateway/pom.xml | 5 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 11 +- .../hadoop/ozone/s3/OzoneServiceProvider.java | 50 +- .../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 44 ++ .../hadoop/ozone/s3/util/TestOzoneS3Util.java | 130 ++++ hadoop-ozone/tools/pom.xml | 9 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +- .../services/org.apache.hadoop.fs.FileSystem} | 11 +- hadoop-ozone/upgrade/pom.xml | 5 +- hadoop-project/pom.xml | 4 +- hadoop-project/src/site/site.xml | 1 + .../java/org/apache/hadoop/fs/s3a/Constants.java | 28 +- .../java/org/apache/hadoop/fs/s3a/Invoker.java | 3 +- .../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 6 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 76 +- .../hadoop/fs/s3a/auth/AwsSignerInitializer.java | 53 ++ .../hadoop/fs/s3a/{ => auth}/SignerManager.java | 80 +- .../auth/delegation/DelegationTokenProvider.java | 20 +- .../hadoop/fs/s3a/commit/CommitConstants.java | 22 +- .../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 3 +- .../hadoop/fs/s3a/select/SelectConstants.java | 2 +- .../apache/hadoop/fs/s3a/select/SelectTool.java | 2 +- .../src/site/markdown/tools/hadoop-aws/index.md | 58 ++ .../fs/s3a/ITestLocatedFileStatusFetcher.java | 31 +- .../hadoop/fs/s3a/ITestS3AFSMainOperations.java | 65 ++ .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 7 + .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 7 +- .../apache/hadoop/fs/s3a/TestSignerManager.java | 130 ---- .../hadoop/fs/s3a/auth/ITestCustomSigner.java | 237 ++++++ .../fs/s3a/auth/ITestRestrictedReadAccess.java | 707 ++++++++++++++++++ .../hadoop/fs/s3a/auth/TestSignerManager.java | 590 +++++++++++++++ .../fs/s3a/commit/ITestCommitOperations.java | 5 +- .../s3a/s3guard/AbstractS3GuardToolTestBase.java | 2 +- .../apache/hadoop/fs/s3a/select/ITestS3Select.java | 4 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 18 + .../fs/adl/live/TestAdlSdkConfiguration.java | 1 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 17 + .../hadoop/fs/azurebfs/AbfsConfiguration.java | 14 + .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 22 +- .../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +- .../fs/azurebfs/constants/ConfigurationKeys.java | 8 + .../constants/FileSystemConfigurations.java | 1 + .../fs/azurebfs/services/AbfsOutputStream.java | 7 +- .../hadoop-azure/src/site/markdown/abfs.md | 17 + .../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 27 +- .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 6 +- .../org/apache/hadoop/tools/DistCpConstants.java | 6 + .../apache/hadoop/tools/mapred/CopyCommitter.java | 10 +- .../org/apache/hadoop/tools/mapred/CopyMapper.java | 3 +- .../tools/mapred/RetriableFileCopyCommand.java | 34 +- .../org/apache/hadoop/tools/util/DistCpUtils.java | 26 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 5 +- .../apache/hadoop/tools/mapred/TestCopyMapper.java | 56 ++ .../tools/mapred/TestRetriableFileCopyCommand.java | 25 +- .../apache/hadoop/tools/util/TestDistCpUtils.java | 45 +- .../tools/util/TestDistCpUtilsWithCombineMode.java | 4 +- .../BlockPlacementPolicyAlwaysSatisfied.java | 5 + .../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 25 + .../hadoop/yarn/sls/conf/SLSConfiguration.java | 3 + .../apache/hadoop/yarn/sls/BaseSLSRunnerTest.java | 4 +- .../org/apache/hadoop/yarn/sls/TestSLSRunner.java | 39 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 41 +- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 22 + .../src/main/proto/YarnCsiAdaptor.proto | 1 + .../main/proto/application_history_client.proto | 1 + .../main/proto/applicationclient_protocol.proto | 1 + .../main/proto/applicationmaster_protocol.proto | 1 + .../src/main/proto/client_SCM_protocol.proto | 1 + .../main/proto/containermanagement_protocol.proto | 1 + .../src/main/proto/server/SCM_Admin_protocol.proto | 1 + .../proto/server/application_history_server.proto | 1 + .../resourcemanager_administration_protocol.proto | 1 + ...arn_server_resourcemanager_service_protos.proto | 1 + .../src/main/proto/yarn_csi_adaptor.proto | 4 +- .../src/main/proto/yarn_protos.proto | 1 + .../src/main/proto/yarn_service_protos.proto | 1 + .../hadoop-yarn-services-core/pom.xml | 23 +- .../src/main/proto/ClientAMProtocol.proto | 1 + .../hadoop-yarn/hadoop-yarn-client/pom.xml | 41 +- .../hadoop/yarn/client/cli/SchedConfCLI.java | 147 +++- .../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 59 +- .../hadoop/yarn/client/cli/TestSchedConfCLI.java | 187 ++++- .../src/test/proto/test_amrm_token.proto | 1 + .../hadoop-yarn/hadoop-yarn-common/pom.xml | 41 +- .../hadoop/yarn/logaggregation/LogToolUtils.java | 60 +- .../src/main/proto/yarn_security_token.proto | 1 + .../src/main/resources/yarn-default.xml | 9 + .../logaggregation/TestAggregatedLogFormat.java | 42 +- .../TestLogAggregationIndexedFileController.java | 31 +- .../pom.xml | 39 +- .../yarn_server_timelineserver_recovery.proto | 1 + .../hadoop-yarn-server-common/pom.xml | 39 +- ...DistributedOpportunisticContainerAllocator.java | 357 +++++++++ .../scheduler/OpportunisticContainerAllocator.java | 347 ++------- .../src/main/proto/ResourceTracker.proto | 1 + .../src/main/proto/SCMUploader.proto | 1 + .../main/proto/collectornodemanager_protocol.proto | 1 + .../proto/distributed_scheduling_am_protocol.proto | 2 +- .../src/main/proto/yarn_server_common_protos.proto | 1 + .../proto/yarn_server_common_service_protos.proto | 1 + .../main/proto/yarn_server_federation_protos.proto | 1 + .../TestOpportunisticContainerAllocator.java | 2 +- .../hadoop-yarn-server-nodemanager/pom.xml | 34 +- .../yarn/server/nodemanager/NodeManager.java | 3 +- .../src/main/proto/LocalizationProtocol.proto | 1 + .../proto/yarn_server_nodemanager_recovery.proto | 1 + .../yarn_server_nodemanager_service_protos.proto | 1 + .../logaggregation/TestLogAggregationService.java | 106 +-- .../scheduler/TestDistributedScheduler.java | 4 +- .../nodemanager/webapp/TestNMWebServices.java | 43 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 69 +- .../server/resourcemanager/ClientRMService.java | 6 +- .../resourcemanager/DefaultAMSProcessor.java | 8 + .../OpportunisticContainerAllocatorAMService.java | 30 +- .../yarn/server/resourcemanager/RMAppManager.java | 7 + .../yarn/server/resourcemanager/RMAuditLogger.java | 53 +- .../scheduler/MutableConfigurationProvider.java | 2 + .../scheduler/SchedulerApplicationAttempt.java | 9 + .../resourcemanager/scheduler/SchedulerUtils.java | 23 + .../capacity/CapacitySchedulerConfiguration.java | 7 + .../scheduler/capacity/LeafQueue.java | 8 +- .../conf/FSSchedulerConfigurationStore.java | 5 + .../capacity/conf/InMemoryConfigurationStore.java | 5 + .../capacity/conf/LeveldbConfigurationStore.java | 7 + .../conf/MutableCSConfigurationProvider.java | 46 +- .../capacity/conf/YarnConfigurationStore.java | 7 + .../capacity/conf/ZKConfigurationStore.java | 5 + .../policy/AbstractComparatorOrderingPolicy.java | 8 +- .../FifoOrderingPolicyWithExclusivePartitions.java | 144 ++++ .../scheduler/policy/IteratorSelector.java} | 37 +- .../scheduler/policy/OrderingPolicy.java | 3 +- .../scheduler/policy/SchedulableEntity.java | 5 + .../server/resourcemanager/webapp/RMWSConsts.java | 3 + .../resourcemanager/webapp/RMWebServices.java | 31 + .../yarn_server_resourcemanager_recovery.proto | 1 + .../server/resourcemanager/TestRMAuditLogger.java | 21 +- .../scheduler/TestSchedulerUtils.java | 142 ++++ .../scheduler/capacity/TestCapacityScheduler.java | 11 +- .../scheduler/capacity/TestLeafQueue.java | 145 +++- .../conf/TestFSSchedulerConfigurationStore.java | 16 + .../conf/TestMutableCSConfigurationProvider.java | 8 + .../capacity/conf/TestZKConfigurationStore.java | 9 + .../scheduler/policy/MockSchedulableEntity.java | 15 +- .../scheduler/policy/TestFairOrderingPolicy.java | 12 +- .../scheduler/policy/TestFifoOrderingPolicy.java | 2 +- .../TestFifoOrderingPolicyForPendingApps.java | 5 +- ...tFifoOrderingPolicyWithExclusivePartitions.java | 244 ++++++ .../security/TestDelegationTokenRenewer.java | 4 + .../TestRMWebServicesConfigurationMutation.java | 13 + .../src/test/proto/test_client_tokens.proto | 1 + .../hadoop-yarn-server-tests/pom.xml | 31 +- .../src/test/proto/test_token.proto | 1 + .../src/site/markdown/NodeManager.md | 8 +- .../src/site/markdown/ResourceManagerRest.md | 81 ++ pom.ozone.xml | 29 +- 520 files changed, 18064 insertions(+), 4052 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org