This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-1880-Decom in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 5547d850fb0d21e25648ce155b3287795399e068 Merge: c09e0a1 e722e4d Author: Márton Elek <e...@apache.org> AuthorDate: Mon Oct 14 16:42:02 2019 +0200 Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom BUILDING.txt | 511 +++++++++++++++++++ LICENSE.txt | 258 ++++++++++ NOTICE.txt | 34 ++ README.txt | 7 + .../checks/build.sh => dev-support/bin/qbt | 7 +- .../build.sh => dev-support/bin/smart-apply-patch | 7 +- .../checks/build.sh => dev-support/bin/test-patch | 7 +- dev-support/bin/yetus-wrapper | 188 +++++++ dev-support/byteman/README.md | 31 ++ dev-support/byteman/hadooprpc.btm | 44 ++ .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +- .../hadoop/hdds/scm/XceiverClientManager.java | 11 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 67 +-- .../hdds/scm/client/ContainerOperationClient.java | 10 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 43 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 + .../hadoop/hdds/scm/storage/CommitWatcher.java | 8 +- .../common/dev-support/findbugsExcludeFile.xml | 5 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +- .../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +- .../hdds/ratis/ContainerCommandRequestMessage.java | 107 ++++ .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 17 +- .../hadoop/hdds/scm/ByteStringConversion.java | 62 +++ .../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 --- .../hadoop/hdds/scm/container/ContainerInfo.java | 2 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 21 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +- .../hdds/security/token/BlockTokenVerifier.java | 4 +- .../security/token/OzoneBlockTokenSelector.java | 8 +- .../authority/PKIProfiles/DefaultProfile.java | 4 +- .../hdds/security/x509/keys/HDDSKeyGenerator.java | 6 +- .../apache/hadoop/hdds/tracing/StringCodec.java | 4 +- .../hadoop/hdds/utils/BackgroundService.java | 9 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 6 +- .../org/apache/hadoop/hdds/utils/LevelDBStore.java | 20 +- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 2 +- .../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 3 +- .../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +- .../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 + .../hadoop/ozone/common/ChecksumByteBuffer.java | 122 +++++ .../ozone/common/PureJavaCrc32ByteBuffer.java | 556 ++++++++++++++++++++ .../ozone/common/PureJavaCrc32CByteBuffer.java | 559 +++++++++++++++++++++ .../helpers/ContainerCommandRequestPBHelper.java | 16 +- .../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +- .../apache/hadoop/ozone/lease/LeaseManager.java | 6 +- .../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 19 +- .../hadoop/ozone/lock/PooledLockFactory.java | 7 +- .../apache/hadoop/ozone/web/utils/JsonUtils.java | 5 +- .../common/src/main/resources/ozone-default.xml | 11 + .../ratis/TestContainerCommandRequestMessage.java | 152 ++++++ .../ozone/common/TestChecksumByteBuffer.java | 102 ++++ .../ozone/container/common/impl/ContainerSet.java | 64 ++- .../container/common/impl/HddsDispatcher.java | 11 +- .../RandomContainerDeletionChoosingPolicy.java | 11 +- ...TopNOrderedContainerDeletionChoosingPolicy.java | 13 +- .../common/statemachine/EndpointStateMachine.java | 4 +- .../CloseContainerCommandHandler.java | 6 +- .../commandhandler/DeleteBlocksCommandHandler.java | 28 +- .../server/ratis/ContainerStateMachine.java | 52 +- .../transport/server/ratis/XceiverServerRatis.java | 5 +- .../container/common/volume/HddsVolumeChecker.java | 14 +- .../common/volume/ThrottledAsyncChecker.java | 8 +- .../container/keyvalue/KeyValueBlockIterator.java | 6 +- .../container/keyvalue/KeyValueContainerCheck.java | 34 +- .../ozone/container/keyvalue/KeyValueHandler.java | 117 +++-- .../container/keyvalue/helpers/ChunkUtils.java | 42 +- .../container/keyvalue/impl/BlockManagerImpl.java | 8 +- .../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +- .../container/keyvalue/impl/ChunkManagerImpl.java | 69 ++- .../keyvalue/interfaces/ChunkManager.java | 2 +- .../background/BlockDeletingService.java | 6 +- .../container/ozoneimpl/ContainerController.java | 6 +- .../container/ozoneimpl/ContainerDataScanner.java | 50 +- .../ozoneimpl/ContainerDataScrubberMetrics.java | 4 +- .../ozoneimpl/ContainerMetadataScanner.java | 19 +- .../ContainerMetadataScrubberMetrics.java | 5 +- .../ozoneimpl/ContainerScrubberConfiguration.java | 17 + .../ozone/container/ozoneimpl/OzoneContainer.java | 8 +- .../container/common/impl/TestContainerSet.java | 18 +- .../container/keyvalue/TestChunkManagerImpl.java | 69 +-- .../keyvalue/TestKeyValueContainerCheck.java | 69 ++- .../ozoneimpl/TestContainerScrubberMetrics.java | 25 +- .../container/ozoneimpl/TestOzoneContainer.java | 23 +- hadoop-hdds/docs/content/beyond/Containers.md | 2 +- hadoop-hdds/docs/content/start/FromSource.md | 2 +- .../hadoop/hdds/server/events/EventQueue.java | 2 +- hadoop-hdds/pom.xml | 1 - .../hadoop/hdds/scm/block/BlockManagerImpl.java | 10 +- .../hdds/scm/block/SCMBlockDeletingService.java | 12 +- .../scm/command/CommandStatusReportHandler.java | 12 +- .../container/AbstractContainerReportHandler.java | 6 +- .../scm/container/ContainerActionsHandler.java | 6 +- .../hdds/scm/container/ContainerStateManager.java | 4 +- .../IncrementalContainerReportHandler.java | 6 +- .../algorithms/SCMContainerPlacementRackAware.java | 6 +- .../scm/container/states/ContainerAttribute.java | 22 +- .../scm/container/states/ContainerStateMap.java | 6 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 4 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 4 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +- .../hdds/scm/server/StorageContainerManager.java | 18 +- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 21 +- .../hdds/scm/cli/container/ListSubcommand.java | 4 +- .../client/io/BlockOutputStreamEntryPool.java | 22 +- .../hadoop/ozone/client/io/KeyInputStream.java | 6 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 97 ++-- .../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 6 +- .../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +- .../hadoop/ozone/om/lock/OzoneManagerLock.java | 139 ++++- .../security/OzoneBlockTokenSecretManager.java | 2 +- .../OzoneDelegationTokenSecretManager.java | 6 +- .../security/OzoneDelegationTokenSelector.java | 8 +- .../hadoop/ozone/security/OzoneSecretManager.java | 6 +- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 79 ++- .../dev-support/checks/_mvn_unit_report.sh | 5 + hadoop-ozone/dev-support/checks/blockade.sh | 2 +- hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 4 +- hadoop-ozone/dev-support/checks/integration.sh | 6 +- hadoop-ozone/dev-support/checks/unit.sh | 2 +- .../dist/src/main/assemblies/ozone-src.xml | 7 +- hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 2 +- .../main/compose/ozone-hdfs/docker-compose.yaml | 6 +- .../dist/src/main/compose/ozone-hdfs/docker-config | 46 -- .../dist/src/main/compose/ozone-mr/common-config | 9 - .../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +- .../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop31/.env | 2 +- .../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop32/.env | 2 +- .../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-om-ha/.env | 2 +- .../main/compose/ozone-om-ha/docker-compose.yaml | 10 +- .../src/main/compose/ozone-om-ha/docker-config | 45 -- .../dist/src/main/compose/ozone-recon/.env | 2 +- .../main/compose/ozone-recon/docker-compose.yaml | 8 +- .../src/main/compose/ozone-recon/docker-config | 47 +- .../dist/src/main/compose/ozone-topology/.env | 2 +- .../compose/ozone-topology/docker-compose.yaml | 12 +- .../src/main/compose/ozone-topology/docker-config | 49 -- hadoop-ozone/dist/src/main/compose/ozone/.env | 2 +- .../src/main/compose/ozone/docker-compose.yaml | 6 +- .../dist/src/main/compose/ozone/docker-config | 45 -- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 + .../dist/src/main/compose/ozoneblockade/.env | 2 +- .../main/compose/ozoneblockade/docker-compose.yaml | 8 +- .../src/main/compose/ozoneblockade/docker-config | 45 -- hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 2 +- .../src/main/compose/ozoneperf/docker-compose.yaml | 10 +- .../dist/src/main/compose/ozoneperf/docker-config | 13 - .../dist/src/main/compose/ozones3-haproxy/.env | 2 +- .../compose/ozones3-haproxy/docker-compose.yaml | 12 +- .../src/main/compose/ozones3-haproxy/docker-config | 48 -- hadoop-ozone/dist/src/main/compose/ozones3/.env | 2 +- .../src/main/compose/ozones3/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozones3/docker-config | 48 -- .../dist/src/main/compose/ozonescripts/.env | 2 +- .../src/main/compose/ozonescripts/docker-config | 7 +- .../dist/src/main/compose/ozonesecure-mr/.env | 2 +- .../compose/ozonesecure-mr/docker-compose.yaml | 11 +- .../src/main/compose/ozonesecure-mr/docker-config | 46 -- .../dist/src/main/compose/ozonesecure/.env | 2 +- .../main/compose/ozonesecure/docker-compose.yaml | 10 +- .../src/main/compose/ozonesecure/docker-config | 53 -- .../dist/src/main/compose/ozonesecure/test.sh | 2 + hadoop-ozone/dist/src/main/dockerbin/entrypoint.sh | 2 +- .../src/main/smoketest/scmcli/pipeline.robot} | 19 +- .../src/test/blockade/ozone/cluster.py | 4 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 13 + .../ozone/container/ContainerTestHelper.java | 11 +- .../container/common/TestBlockDeletingService.java | 24 +- .../common/impl/TestContainerPersistence.java | 53 +- .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 4 - .../apache/hadoop/ozone/om/BucketManagerImpl.java | 19 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 66 ++- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 59 +-- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 10 - .../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++- .../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +- .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 28 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +- .../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 12 +- .../om/request/bucket/OMBucketDeleteRequest.java | 16 +- .../request/bucket/OMBucketSetPropertyRequest.java | 13 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 6 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 4 +- .../om/request/file/OMDirectoryCreateRequest.java | 4 +- .../ozone/om/request/file/OMFileCreateRequest.java | 4 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 4 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 4 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 4 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +- .../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +- .../request/s3/bucket/S3BucketCreateRequest.java | 22 +- .../request/s3/bucket/S3BucketDeleteRequest.java | 13 +- .../S3InitiateMultipartUploadRequest.java | 4 +- .../multipart/S3MultipartUploadAbortRequest.java | 4 +- .../S3MultipartUploadCommitPartRequest.java | 4 +- .../S3MultipartUploadCompleteRequest.java | 4 +- .../om/request/s3/security/S3GetSecretRequest.java | 6 +- .../om/request/volume/OMVolumeCreateRequest.java | 10 +- .../om/request/volume/OMVolumeDeleteRequest.java | 10 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 4 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 6 +- .../OzoneManagerHARequestHandlerImpl.java | 4 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +- .../protocolPB/OzoneManagerRequestHandler.java | 4 +- .../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +- .../hadoop/ozone/web/ozShell/ObjectPrinter.java | 3 +- .../web/ozShell/bucket/AddAclBucketHandler.java | 5 +- .../web/ozShell/bucket/GetAclBucketHandler.java | 4 +- .../web/ozShell/bucket/RemoveAclBucketHandler.java | 7 +- .../web/ozShell/bucket/SetAclBucketHandler.java | 5 +- .../ozone/web/ozShell/keys/AddAclKeyHandler.java | 5 +- .../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +- .../web/ozShell/keys/RemoveAclKeyHandler.java | 7 +- .../ozone/web/ozShell/keys/SetAclKeyHandler.java | 5 +- .../ozone/web/ozShell/token/GetTokenHandler.java | 2 +- .../ozone/web/ozShell/token/PrintTokenHandler.java | 2 +- .../web/ozShell/volume/AddAclVolumeHandler.java | 5 +- .../web/ozShell/volume/GetAclVolumeHandler.java | 4 +- .../web/ozShell/volume/RemoveAclVolumeHandler.java | 7 +- .../web/ozShell/volume/SetAclVolumeHandler.java | 5 +- .../hadoop/ozone/om/TestKeyDeletingService.java | 3 + .../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++ .../ozone/om/request/TestOMRequestUtils.java | 60 ++- hadoop-ozone/ozonefs-lib-current/pom.xml | 3 + .../fs/ozone/BasicOzoneClientAdapterImpl.java | 59 +-- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +- hadoop-ozone/pom.xml | 1 - .../org/apache/hadoop/ozone/recon/ReconUtils.java | 61 +++ .../apache/hadoop/ozone/recon/TestReconUtils.java | 44 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 6 +- .../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 5 +- .../ozone/s3/exception/OS3ExceptionMapper.java | 4 +- pom.ozone.xml => pom.xml | 0 253 files changed, 5036 insertions(+), 1757 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: hdfs-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: hdfs-commits-h...@hadoop.apache.org