[hadoop] 01/01: Merge branch 'trunk' into ozone-0.4.1
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a commit to branch ozone-0.4.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 20b2cd669527b152a2750c86d0ed9c6cc6a31846 Merge: 0b92dec b41ef61 Author: Anu Engineer AuthorDate: Thu Jul 25 14:08:41 2019 -0700 Merge branch 'trunk' into ozone-0.4.1 .../org/apache/hadoop/fs/shell/AclCommands.java| 6 +- .../main/java/org/apache/hadoop/io/IOUtils.java| 2 - .../src/main/resources/core-default.xml| 4 +- .../src/site/markdown/FileSystemShell.md | 2 +- .../src/site/markdown/filesystem/filesystem.md | 44 ++- .../filesystem/fsdatainputstreambuilder.md | 2 +- .../AbstractContractRootDirectoryTest.java | 2 +- .../hadoop/fs/contract/ContractTestUtils.java | 30 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 4 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../common/src/main/resources/ozone-default.xml| 8 + .../hadoop/hdds/scm/cli/TopologySubcommand.java| 60 +++- .../server/federation/router/FederationUtil.java | 13 +- .../federation/router/TestFederationUtil.java | 69 .../hdfs/server/namenode/FSImageFormatPBINode.java | 4 +- .../hdfs/server/namenode/INodeDirectory.java | 16 + .../hadoop/hdfs/server/namenode/NameNode.java | 4 + .../hadoop/hdfs/server/common/TestJspHelper.java | 34 ++ .../test/java/org/apache/hadoop/hdfs/NNBench.java | 2 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 - .../org/apache/hadoop/ozone/audit/OMAction.java| 2 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 3 + .../hadoop/ozone/om/exceptions/OMException.java| 3 +- .../ozone/om/protocol/OzoneManagerHAProtocol.java | 18 +- .../src/main/proto/OzoneManagerProtocol.proto | 9 +- .../dist/src/main/compose/ozone-topology/test.sh | 2 + .../dist/src/main/smoketest/topology/scmcli.robot | 32 ++ .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 7 + .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 54 ++- .../ozoneimpl/TestSecureOzoneContainer.java| 2 + .../server/TestSecureContainerServer.java | 2 + .../hadoop/ozone/om/TestOMRatisSnapshots.java | 189 ++ .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 7 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 11 - .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 10 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 2 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 22 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 381 +++-- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 3 + .../ozone/om/ratis/OzoneManagerRatisServer.java| 15 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 130 +++ .../om/ratis/utils/OzoneManagerRatisUtils.java | 12 + .../request/s3/bucket/S3BucketDeleteRequest.java | 193 +++ .../S3InitiateMultipartUploadRequest.java | 213 .../multipart/S3MultipartUploadAbortRequest.java | 173 ++ .../S3MultipartUploadCommitPartRequest.java| 217 .../om/request/s3/multipart/package-info.java} | 21 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 4 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 3 +- .../response/s3/bucket/S3BucketCreateResponse.java | 8 +- ...teResponse.java => S3BucketDeleteResponse.java} | 54 +-- .../S3InitiateMultipartUploadResponse.java | 80 + .../multipart/S3MultipartUploadAbortResponse.java | 83 + .../S3MultipartUploadCommitPartResponse.java | 109 ++ .../om/response/s3/multipart/package-info.java}| 22 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +- .../OzoneManagerHARequestHandlerImpl.java | 18 +- .../protocolPB/OzoneManagerRequestHandler.java | 35 -- .../ozone/om/request/TestOMRequestUtils.java | 85 + .../s3/bucket/TestS3BucketDeleteRequest.java | 167 + .../TestS3InitiateMultipartUploadRequest.java | 153 + .../s3/multipart/TestS3MultipartRequest.java | 178 ++ .../TestS3MultipartUploadAbortRequest.java | 158 + .../TestS3MultipartUploadCommitPartRequest.java| 209 +++ .../s3/multipart/package-info.java}| 20 +- .../ozone/om/response/TestOMResponseUtils.java | 42 +++ .../s3/bucket/TestS3BucketCreateResponse.java | 41 +-- .../s3/bucket/TestS3BucketDeleteResponse.java | 73 .../TestS3InitiateMultipartUploadResponse.java | 63 .../s3/multipart/TestS3MultipartResponse.java | 143 .../TestS3MultipartUploadAbortResponse.java| 129 +++ .../multipart/package-info.java} | 20 +- hadoop-project/pom.xml | 3 +- .../java/org/apache/hadoop/fs/s3a/Constants.java | 4
[hadoop] 01/01: Merge branch 'trunk' into ozone-0.4.1. This is the second merge to pick up a NPE fix in Trunk.
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit fc3bfdc53523f4273d13f2c532a8689e1f2263bf
Merge: e1e3701 d5ef38b
Author: Anu Engineer
AuthorDate: Thu Jul 18 14:28:52 2019 -0700
Merge branch 'trunk' into ozone-0.4.1. This is the second merge to pick up
a NPE fix in Trunk.
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 1 +
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 5 +
.../java/org/apache/hadoop/utils/db/DBStore.java | 14 +-
.../java/org/apache/hadoop/utils/db/RDBStore.java | 9 +
.../org/apache/hadoop/utils/db/TypedTable.java | 86 -
.../apache/hadoop/utils/db/cache/CacheResult.java | 58
.../apache/hadoop/utils/db/cache/TableCache.java | 31 +-
...{PartialTableCache.java => TableCacheImpl.java} | 79 -
...tialTableCache.java => TestTableCacheImpl.java} | 71 ++--
.../container/common/impl/ContainerDataYaml.java | 9 +-
.../server/ratis/ContainerStateMachine.java| 31 +-
.../transport/server/ratis/XceiverServerRatis.java | 55 ++--
.../placement/algorithms/SCMCommonPolicy.java | 4 +-
.../algorithms/TestContainerPlacementFactory.java | 25 +-
.../src/main/resources/hdfs-rbf-default.xml| 2 +-
.../hdfs/server/federation/router/TestRouter.java | 28 ++
.../hadoop/hdfs/server/namenode/ImageServlet.java | 63 +++-
.../hadoop/hdfs/server/namenode/NameNode.java | 6 +
.../server/namenode/ha/StandbyCheckpointer.java| 146 ++---
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 8 +-
.../src/site/markdown/HdfsQuotaAdminGuide.md | 4 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 5 +
.../hdfs/server/namenode/TestCheckpoint.java | 55
.../server/namenode/ha/TestStandbyCheckpoints.java | 35 +-
.../src/test/resources/testHDFSConf.xml| 8 +-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 4 +-
.../org/apache/hadoop/ozone/audit/OMAction.java| 3 +
.../hadoop/ozone/om/exceptions/OMException.java| 2 +
.../ozone/om/helpers/OmDeleteVolumeResponse.java | 49 ---
.../om/helpers/OmVolumeOwnerChangeResponse.java| 56
.../ozone/om/protocol/OzoneManagerHAProtocol.java | 77 -
.../src/main/proto/OzoneManagerProtocol.proto | 20 +-
.../ozone/client/rpc/Test2WayCommitInRatis.java| 156 +
.../client/rpc/TestFailureHandlingByClient.java| 52 +--
.../rpc/TestMultiBlockWritesWithDnFailures.java| 168 ++
.../ozone/client/rpc/TestWatchForCommit.java | 100 ++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 6 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 3 -
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 24 ++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 7 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 77 -
.../hadoop/ozone/om/S3BucketManagerImpl.java | 13 +-
.../org/apache/hadoop/ozone/om/VolumeManager.java | 50 +--
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 102 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 1 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 11 +
.../request/s3/bucket/S3BucketCreateRequest.java | 361 +
.../ozone/om/request/s3/bucket/package-info.java | 23 ++
.../om/request/volume/OMVolumeCreateRequest.java | 36 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 4 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 45 ++-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 4 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 3 +-
.../response/s3/bucket/S3BucketCreateResponse.java | 73 +
.../ozone/om/response/s3/bucket/package-info.java | 24 ++
.../protocolPB/OzoneManagerHARequestHandler.java | 11 -
.../OzoneManagerHARequestHandlerImpl.java | 169 +-
.../ozone/om/request/TestOMRequestUtils.java | 20 ++
.../s3/bucket/TestS3BucketCreateRequest.java | 246 ++
.../ozone/om/request/s3/bucket/package-info.java | 23 ++
.../request/volume/TestOMVolumeCreateRequest.java | 27 +-
.../request/volume/TestOMVolumeDeleteRequest.java | 18 +-
.../s3/bucket/TestS3BucketCreateResponse.java | 119 +++
.../ozone/om/response/s3/bucket/package-info.java | 23 ++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 41 +--
.../apache/hadoop/yarn/server/MiniYARNCluster.java | 8 +-
.../src/site/markdown/YarnApplicationSecurity.md | 8 +-
70 files changed, 2139 insertions(+), 978 deletions(-)
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional
[hadoop] 01/01: Merge branch 'trunk' into ozone-0.4.1
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a commit to branch ozone-0.4.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit e1e3701f3582693621974913eb54a610079acb5d Merge: 669c447 eb0849b Author: Anu Engineer AuthorDate: Thu Jul 18 13:42:37 2019 -0700 Merge branch 'trunk' into ozone-0.4.1 .../org/apache/hadoop/net/NetworkTopology.java | 9 +- .../org/apache/hadoop/security/token/Token.java| 29 +- .../service/launcher/ServiceLaunchException.java | 14 + .../hadoop/service/launcher/ServiceLauncher.java | 70 +- .../apache/hadoop/util/ShutdownHookManager.java| 21 +- .../hadoop-common/src/main/proto/Security.proto| 1 - .../src/main/resources/core-default.xml| 2 +- .../AbstractContractRootDirectoryTest.java | 71 +- .../hadoop/fs/contract/ContractTestUtils.java | 4 +- .../hadoop/util/TestShutdownHookManager.java | 29 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 38 +- .../hadoop/hdds/scm/XceiverClientManager.java | 68 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 4 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 25 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 28 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +- .../hadoop/hdds/scm/net/NetworkTopology.java | 4 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 53 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 ++- .../scm/protocol/ScmBlockLocationProtocol.java | 9 + ...lockLocationProtocolClientSideTranslatorPB.java | 34 + .../hdds/scm/storage/ContainerProtocolCalls.java | 4 +- .../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +- ...lockLocationProtocolServerSideTranslatorPB.java | 26 + .../src/main/proto/ScmBlockLocationProtocol.proto | 16 + hadoop-hdds/common/src/main/proto/hdds.proto | 5 +- .../common/src/main/resources/ozone-default.xml| 12 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 35 +- .../container/common/interfaces/Container.java | 4 +- .../ozone/container/common/interfaces/Handler.java | 5 + .../container/common/report/ReportManager.java | 10 + .../states/endpoint/HeartbeatEndpointTask.java | 6 +- .../common/transport/server/XceiverServerGrpc.java | 6 + .../common/transport/server/ratis/CSMMetrics.java | 30 + .../server/ratis/ContainerStateMachine.java| 30 +- .../transport/server/ratis/XceiverServerRatis.java | 30 +- .../container/keyvalue/KeyValueContainer.java | 20 +- .../container/keyvalue/KeyValueContainerCheck.java | 46 +- .../ozone/container/keyvalue/KeyValueHandler.java | 18 +- .../container/ozoneimpl/ContainerController.java | 16 +- .../container/ozoneimpl/ContainerScrubber.java | 31 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 1 + .../proto/StorageContainerDatanodeProtocol.proto | 1 + .../keyvalue/TestKeyValueContainerCheck.java | 11 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java| 1 + .../IncrementalContainerReportHandler.java | 2 + .../ContainerPlacementPolicyFactory.java | 6 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 17 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 2 + .../hdds/scm/pipeline/SCMPipelineManager.java | 10 +- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 56 +- .../hdds/scm/server/SCMBlockProtocolServer.java| 37 + .../hadoop/hdds/scm/server/SCMConfigurator.java| 19 + .../hdds/scm/server/StorageContainerManager.java | 14 +- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 4 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 14 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 10 + .../scm/server/TestSCMBlockProtocolServer.java | 153 .../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 20 + .../apache/hadoop/hdfs/DistributedFileSystem.java | 8 +- .../datatransfer/sasl/SaslDataTransferClient.java | 55 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 12 +- .../security/token/block/BlockTokenIdentifier.java | 32 + .../hadoop-hdfs-client/src/main/proto/hdfs.proto | 1 + .../server/federation/router/RBFConfigKeys.java| 8 + .../federation/router/RouterPermissionChecker.java | 2 +- .../server/federation/store/CachedRecordStore.java | 22 +- .../server/federation/store/StateStoreService.java | 16 +- .../federation/store/records/BaseRecord.java | 46 +- .../federation/store/records/MembershipState.java | 21 + .../federation/store/records/QueryResult.java | 3 +- .../federation/store/records/RouterState.java | 17 + .../records/impl/pb/MembershipStatePBImpl.java | 4 +- .../store/records/impl/pb/RouterStatePBImpl.java | 4 +- .../src/main/resources/hdfs-rbf-default.xml| 20 +
