This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-2823 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a3199cc534532fb28f63783571ce8a5daed891db Merge: 8791179 1c6a257 Author: Nandakumar <[email protected]> AuthorDate: Fri Jun 5 20:07:37 2020 +0530 Merge branch 'master' into HDDS-2823 .github/coverage-report.sh | 56 + .github/workflows/post-commit.yml | 40 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 7 +- .../common/dev-support/findbugsExcludeFile.xml | 3 - hadoop-hdds/common/pom.xml | 92 +- .../java/org/apache/hadoop/hdds/StringUtils.java | 77 +- .../hadoop/hdds/scm/ContainerPlacementStatus.java | 14 + .../java/org/apache/hadoop/hdds/utils/IOUtils.java | 54 + .../org/apache/hadoop/hdds/utils/SignalLogger.java | 93 ++ .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 14 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 +- .../common/src/main/resources/ozone-default.xml | 26 +- .../hadoop/hdds/conf/SimpleConfiguration.java | 3 +- .../hdds/conf/ConfigurationReflectionUtil.java | 8 +- .../org/apache/hadoop/hdds/conf/PostConstruct.java | 28 +- .../dev-support/findbugsExcludeFile.xml | 3 - hadoop-hdds/container-service/pom.xml | 30 - .../apache/hadoop/ozone/HddsDatanodeService.java | 9 +- .../container/common/impl/ChunkLayOutVersion.java | 22 +- .../container/common/impl/HddsDispatcher.java | 46 +- .../common/statemachine/DatanodeConfiguration.java | 43 +- .../commandhandler/DeleteBlocksCommandHandler.java | 6 +- .../states/endpoint/HeartbeatEndpointTask.java | 5 +- .../ozone/container/keyvalue/KeyValueHandler.java | 7 +- .../container/keyvalue/helpers/ChunkUtils.java | 9 + .../keyvalue/impl/ChunkManagerDispatcher.java | 6 +- .../keyvalue/impl/FilePerBlockStrategy.java | 12 +- .../keyvalue/interfaces/ChunkManager.java | 4 +- .../background/BlockDeletingService.java | 10 +- .../ozone/container/common/SCMTestUtils.java | 4 +- .../container/common/TestBlockDeletingService.java | 4 +- .../statemachine/TestDatanodeConfiguration.java | 85 + .../keyvalue/TestKeyValueBlockIterator.java | 4 +- .../keyvalue/TestKeyValueContainerCheck.java | 6 +- .../container/ozoneimpl/TestContainerReader.java | 4 +- .../docs/content/recipe/SparkOzoneFSK8S.zh.md | 185 +++ hadoop-hdds/docs/content/shell/KeyCommands.md | 20 + hadoop-hdds/docs/content/shell/KeyCommands.zh.md | 22 +- hadoop-hdds/docs/static/ozone-logo-monochrome.svg | 195 +++ hadoop-hdds/framework/pom.xml | 8 + .../freon/FakeScmBlockLocationProtocolClient.java | 0 .../FakeScmContainerLocationProtocolClient.java | 0 .../org/apache/hadoop/hdds/freon/package-info.java | 22 +- .../common/helpers/DeleteBlockResult.java | 0 .../scm/container/common/helpers/package-info.java | 30 +- .../scm/protocol/ScmBlockLocationProtocol.java | 0 .../hadoop/hdds/scm/protocol/package-info.java | 27 +- ...lockLocationProtocolClientSideTranslatorPB.java | 0 .../scm/protocolPB/ScmBlockLocationProtocolPB.java | 0 ...inerLocationProtocolClientSideTranslatorPB.java | 0 .../StorageContainerLocationProtocolPB.java | 0 .../hadoop/hdds/scm/protocolPB/package-info.java | 26 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 15 + .../org/apache/hadoop/ozone/common/BlockGroup.java | 0 .../ozone/common/DeleteBlockGroupResult.java | 0 .../apache/hadoop/ozone/common}/package-info.java | 16 +- .../exceptions/TestSCMExceptionResultCodes.java | 0 .../dev-support/findbugsExcludeFile.xml | 3 +- hadoop-hdds/interface-admin/pom.xml | 82 + .../src/main/proto/ScmAdminProtocol.proto} | 0 .../interface-admin/src/main/proto/proto.lock | 895 ++++++++++ .../dev-support/findbugsExcludeFile.xml | 6 +- hadoop-hdds/interface-client/pom.xml | 135 ++ .../src/main/proto/DatanodeClientProtocol.proto} | 0 .../src/main/proto/hdds.proto | 0 .../src/main/proto/proto.lock | 1748 +------------------- .../dev-support/findbugsExcludeFile.xml | 3 +- hadoop-hdds/interface-server/pom.xml | 82 + .../ScmServerDatanodeHeartbeatProtocol.proto} | 0 .../src/main/proto/ScmServerProtocol.proto} | 0 .../main/proto/ScmServerSecurityProtocol.proto} | 0 .../src/main/proto/proto.lock | 714 +++++++- hadoop-hdds/pom.xml | 21 + .../ContainerPlacementStatusDefault.java | 10 + .../hdds/scm/server/StorageContainerManager.java | 3 +- .../scm/server/StorageContainerManagerStarter.java | 7 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 4 +- .../hadoop/hdds/scm/cli/TopologySubcommand.java | 11 +- .../ozone/client/protocol/ClientProtocol.java | 5 + .../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 6 +- .../common/dev-support/findbugsExcludeFile.xml | 3 + hadoop-ozone/common/pom.xml | 7 +- .../common/src/main/proto/Security.proto | 0 hadoop-ozone/common/src/main/proto/proto.lock | 143 ++ .../org/apache/hadoop/ozone/csi/CsiServer.java | 7 +- hadoop-ozone/dev-support/checks/sonar.sh | 12 +- hadoop-ozone/dist/pom.xml | 4 +- .../compose/ozone-mr/hadoop27/docker-compose.yaml | 4 +- .../main/compose/ozone-mr/hadoop27/docker-config | 4 +- .../ozone-mr/hadoop27/{run-test.sh => test.sh} | 0 .../compose/ozone-mr/hadoop31/docker-compose.yaml | 4 +- .../main/compose/ozone-mr/hadoop31/docker-config | 2 +- .../compose/ozone-mr/hadoop32/docker-compose.yaml | 4 +- .../main/compose/ozone-mr/hadoop32/docker-config | 2 +- .../compose/ozonesecure-mr/docker-compose.yaml | 6 +- .../src/main/compose/ozonesecure-mr/docker-config | 2 +- hadoop-ozone/dist/src/main/smoketest/csi.robot | 2 +- .../dist/src/main/smoketest/mapreduce.robot | 12 +- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 5 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 29 + .../ozone/TestStorageContainerManagerHelper.java | 4 +- .../ozone/client/rpc/TestBlockOutputStream.java | 2 + ...m.java => TestBlockOutputStreamFlushDelay.java} | 62 +- .../rpc/TestBlockOutputStreamWithFailures.java | 2 + ...stBlockOutputStreamWithFailuresFlushDelay.java} | 33 +- .../client/rpc/TestContainerStateMachine.java | 8 +- .../rpc/TestContainerStateMachineFailures.java | 4 +- ...va => TestContainerStateMachineFlushDelay.java} | 107 +- .../client/rpc/TestFailureHandlingByClient.java | 3 +- .../rpc/TestFailureHandlingByClientFlushDelay.java | 235 +++ .../rpc/TestOzoneClientRetriesOnException.java | 2 + ...stOzoneClientRetriesOnExceptionFlushDelay.java} | 148 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 18 +- .../commandhandler/TestBlockDeletion.java | 8 +- .../ozone/freon/TestHadoopDirTreeGenerator.java | 180 ++ .../ozone/om/TestOzoneManagerConfiguration.java | 106 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 6 +- hadoop-ozone/ozone-manager/pom.xml | 6 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 10 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 215 +-- .../hadoop/ozone/om/OzoneManagerStarter.java | 9 +- .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 86 +- .../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 18 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 13 +- hadoop-ozone/{ozonefs => ozonefs-common}/pom.xml | 72 +- .../org/apache/hadoop/fs/ozone/BasicKeyInfo.java | 0 .../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 0 .../fs/ozone/BasicOzoneClientAdapterImpl.java | 0 .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 66 +- .../hadoop/fs/ozone/CapableOzoneFSInputStream.java | 0 .../java/org/apache/hadoop/fs/ozone/Constants.java | 0 .../apache/hadoop/fs/ozone/FileStatusAdapter.java | 0 .../org/apache/hadoop/fs/ozone/O3fsDtFetcher.java | 0 .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 0 .../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 0 .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 0 .../hadoop/fs/ozone/OzoneFSOutputStream.java | 0 .../hadoop/fs/ozone/OzoneFSStorageStatistics.java | 0 .../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 0 .../hadoop/fs/ozone/OzoneStreamCapabilities.java | 0 .../java/org/apache/hadoop/fs/ozone/Statistic.java | 0 .../org/apache/hadoop/fs/ozone/package-info.java} | 30 +- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 0 .../org/apache/hadoop/fs/ozone/package-info.java | 0 .../services/org.apache.hadoop.fs.FileSystem | 0 .../src/test/resources/log4j.properties | 0 hadoop-ozone/ozonefs-hadoop2/pom.xml | 105 ++ .../fs/ozone/Hadoop27OmTransportFactory.java} | 30 +- .../hadoop/fs/ozone/Hadoop27RpcTransport.java | 82 + .../java/org/apache/hadoop/fs/ozone/OzFs.java} | 18 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java} | 17 +- .../org/apache/hadoop/fs/ozone/package-info.java} | 30 +- .../services/org.apache.hadoop.fs.FileSystem | 0 ....hadoop.ozone.om.protocolPB.OmTransportFactory} | 2 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 85 + .../java/org/apache/hadoop/fs/ozone/OzFs.java} | 18 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 33 +- .../org/apache/hadoop/fs/ozone/package-info.java} | 30 +- .../services/org.apache.hadoop.fs.FileSystem | 0 hadoop-ozone/ozonefs-lib-legacy/pom.xml | 138 -- .../src/main/resources/ozonefs.txt | 21 - .../pom.xml | 122 +- hadoop-ozone/ozonefs/pom.xml | 42 +- .../hadoop/fs/ozone/FilteredClassLoader.java | 96 -- .../hadoop/fs/ozone/OzoneClientAdapterFactory.java | 169 -- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 33 +- .../services/org.apache.hadoop.fs.FileSystem | 0 .../hadoop/fs/ozone/TestFilteredClassLoader.java | 63 - .../fs/ozone/TestOzoneFileSystemWithMocks.java | 171 -- .../hadoop/fs/ozone/TestReadWriteStatistics.java | 463 ------ hadoop-ozone/pom.xml | 25 +- .../recon/schema/ContainerSchemaDefinition.java | 41 +- .../recon/schema/UtilizationSchemaDefinition.java | 19 +- .../apache/hadoop/ozone/recon/ReconConstants.java | 3 + .../hadoop/ozone/recon/ReconControllerModule.java | 7 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 7 + .../hadoop/ozone/recon/api/ContainerEndpoint.java | 2 +- .../ozone/recon/api/UtilizationEndpoint.java | 101 ++ .../hadoop/ozone/recon/api/UtilizationService.java | 52 - .../ozone/recon/fsck/ContainerHealthStatus.java | 118 ++ .../ozone/recon/fsck/ContainerHealthTask.java | 375 +++++ .../ozone/recon/fsck/MissingContainerTask.java | 113 -- .../recon/persistence/ContainerSchemaManager.java | 38 +- .../scm/ReconStorageContainerManagerFacade.java | 14 +- .../recon/spi/ContainerDBServiceProvider.java | 4 +- .../spi/impl/ContainerDBServiceProviderImpl.java | 4 +- .../ozone/recon/tasks/FileSizeCountTask.java | 213 +-- .../ozone/recon/api/TestContainerEndpoint.java | 16 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 122 ++ .../recon/fsck/TestContainerHealthStatus.java | 181 ++ .../ozone/recon/fsck/TestContainerHealthTask.java | 280 ++++ .../TestContainerHealthTaskRecordGenerator.java | 345 ++++ .../ozone/recon/fsck/TestMissingContainerTask.java | 120 -- .../TestUtilizationSchemaDefinition.java | 26 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 339 +++- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 7 + .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 12 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 1 + .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 221 +++ .../hadoop/ozone/freon/RandomKeyGenerator.java | 4 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 9 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 24 +- 203 files changed, 6873 insertions(+), 4594 deletions(-) diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 6bf5da5,4b4e073..6342f88 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -103,9 -90,9 +103,8 @@@ import org.apache.hadoop.hdds.security. import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; - import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.metrics2.MetricsSystem; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
