This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f904539c0fa0798f0fbc4c09e5ae98bf06d1a91e Merge: c960ec7 69b92a0 Author: Elek Márton <[email protected]> AuthorDate: Wed Jun 2 11:35:59 2021 +0200 Merge remote-tracking branch 'origin/master' into HDDS-3816-ec .github/workflows/post-commit.yml | 26 +- hadoop-hdds/client/pom.xml | 30 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 29 +- .../hadoop/hdds/scm/storage/ByteArrayReader.java | 67 ++++ .../hadoop/hdds/scm/storage/ByteBufferReader.java | 71 +++++ .../hdds/scm/storage/ByteReaderStrategy.java | 41 +++ .../hadoop/hdds/scm/storage/ChunkInputStream.java | 38 ++- .../hdds/scm/storage/TestBlockInputStream.java | 35 +++ hadoop-hdds/common/pom.xml | 26 +- .../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 5 - .../java/org/apache/hadoop/hdds/HddsUtils.java | 17 + hadoop-hdds/config/pom.xml | 10 +- hadoop-hdds/container-service/pom.xml | 42 +-- .../container/common/report/ReportPublisher.java | 4 +- .../common/statemachine/DatanodeConfiguration.java | 30 ++ .../common/statemachine/DatanodeStateMachine.java | 6 + .../CloseContainerCommandHandler.java | 9 +- .../states/endpoint/HeartbeatEndpointTask.java | 4 + .../container/common/volume/HddsVolumeChecker.java | 11 +- .../container/common/volume/MutableVolumeSet.java | 42 ++- .../container/keyvalue/helpers/ChunkUtils.java | 12 + .../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +- .../keyvalue/impl/FilePerBlockStrategy.java | 5 +- .../keyvalue/impl/FilePerChunkStrategy.java | 2 + .../ozone/container/ozoneimpl/OzoneContainer.java | 5 +- .../hadoop/ozone/protocol/commands/SCMCommand.java | 10 + .../container/common/TestBlockDeletingService.java | 2 +- .../TestSchemaOneBackwardsCompatibility.java | 4 +- .../common/impl/TestContainerPersistence.java | 2 +- .../container/common/impl/TestHddsDispatcher.java | 5 +- .../common/report/TestReportPublisher.java | 9 +- .../statemachine/TestDatanodeConfiguration.java | 12 + .../container/common/volume/TestVolumeSet.java | 5 +- .../common/volume/TestVolumeSetDiskChecks.java | 11 +- .../keyvalue/TestKeyValueBlockIterator.java | 2 +- .../keyvalue/TestKeyValueContainerCheck.java | 2 +- .../container/keyvalue/TestKeyValueHandler.java | 3 +- .../keyvalue/impl/CommonChunkManagerTestCases.java | 33 ++ .../container/ozoneimpl/TestContainerReader.java | 2 +- .../container/ozoneimpl/TestOzoneContainer.java | 3 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 68 +++- hadoop-hdds/docs/content/feature/scm-secure-ha.png | Bin 0 -> 46757 bytes hadoop-hdds/docs/content/interface/O3fs.md | 4 +- hadoop-hdds/docs/content/interface/O3fs.zh.md | 8 +- hadoop-hdds/docs/content/interface/Ofs.md | 4 +- hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 6 +- .../docs/content/recipe/SparkOzoneFSK8S.zh.md | 6 +- hadoop-hdds/docs/pom.xml | 6 +- hadoop-hdds/framework/pom.xml | 34 +- ...etManager.java => ContainerTokenGenerator.java} | 58 ++-- .../token/ContainerTokenSecretManager.java | 30 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 6 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 6 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 6 +- hadoop-hdds/interface-admin/pom.xml | 10 +- hadoop-hdds/interface-client/pom.xml | 6 +- hadoop-hdds/interface-server/pom.xml | 10 +- .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 + hadoop-hdds/pom.xml | 78 ++--- hadoop-hdds/server-scm/pom.xml | 56 ++-- .../scm/container/CloseContainerEventHandler.java | 9 + .../hdds/scm/container/ReplicationManager.java | 12 +- .../scm/container/balancer/ContainerBalancer.java | 341 ++++++++++++++++++--- .../balancer/ContainerBalancerConfiguration.java | 52 ++-- .../balancer/ContainerBalancerMetrics.java | 127 ++++++++ .../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 13 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 1 + .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 29 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 29 +- .../hdds/scm/server/SCMClientProtocolServer.java | 17 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 4 +- .../apache/hadoop/hdds/scm/server/SCMMXBean.java | 10 + .../hdds/scm/server/StorageContainerManager.java | 52 +++- .../main/resources/webapps/scm/scm-overview.html | 8 + .../hadoop/hdds/scm/container/MockNodeManager.java | 88 ++++-- .../container/balancer/TestContainerBalancer.java | 219 +++++++++++++ .../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 6 +- .../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 4 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 4 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 25 ++ .../hdds/scm/safemode/TestSCMSafeModeManager.java | 2 +- hadoop-hdds/test-utils/pom.xml | 6 +- hadoop-hdds/tools/pom.xml | 26 +- .../hdds/scm/cli/ContainerOperationClient.java | 42 ++- .../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 14 +- hadoop-ozone/client/pom.xml | 14 +- .../hadoop/ozone/client/io/KeyInputStream.java | 51 ++- .../hadoop/ozone/client/io/OzoneInputStream.java | 15 +- hadoop-ozone/common/pom.xml | 26 +- .../hadoop/ozone/client/io/LengthInputStream.java | 17 +- hadoop-ozone/csi/pom.xml | 14 +- hadoop-ozone/datanode/pom.xml | 14 +- hadoop-ozone/dev-support/checks/coverage.sh | 3 +- hadoop-ozone/dev-support/checks/integration.sh | 2 +- hadoop-ozone/dev-support/checks/unit.sh | 2 +- hadoop-ozone/dev-support/docker/Dockerfile | 2 +- .../intellij/runConfigurations/CsiServer.xml | 4 +- .../intellij/runConfigurations/Datanode1.xml | 2 +- .../intellij/runConfigurations/Datanode2.xml | 2 +- .../intellij/runConfigurations/Datanode3.xml | 2 +- .../intellij/runConfigurations/FreonStandalone.xml | 4 +- .../intellij/runConfigurations/OzoneFsShell.xml | 2 +- .../intellij/runConfigurations/OzoneManager.xml | 2 +- .../runConfigurations/OzoneManagerInit.xml | 2 +- .../intellij/runConfigurations/OzoneShell.xml | 2 +- .../intellij/runConfigurations/Recon.xml | 4 +- .../intellij/runConfigurations/S3Gateway.xml | 4 +- .../runConfigurations/StorageContainerManager.xml | 2 +- .../StorageContainerManagerInit.xml | 2 +- .../dist/dev-support/bin/dist-layout-stitching | 3 + .../dist/dev-support/bin/dist-tar-stitching | 6 +- hadoop-ozone/dist/pom.xml | 73 ++--- .../docker-image/docker-krb5/Dockerfile-krb5 | 35 --- .../common/docker-image/docker-krb5/README.md | 34 -- .../common/docker-image/docker-krb5/kadm5.acl | 20 -- .../common/docker-image/docker-krb5/launcher.sh | 25 -- .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../main/compose/ozone-mr/hadoop27/docker-config | 4 +- .../main/compose/ozone-mr/hadoop31/docker-config | 4 +- .../main/compose/ozone-mr/hadoop32/docker-config | 4 +- .../dist/src/main/compose/ozonesecure-ha/.env | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 44 ++- .../src/main/compose/ozonesecure-ha/docker-config | 24 +- .../compose/ozonesecure-ha/keytabs/HTTP.keytab | Bin 0 -> 144 bytes .../main/compose/ozonesecure-ha/keytabs/dn.keytab | Bin 0 -> 278 bytes .../main/compose/ozonesecure-ha/keytabs/om.keytab | Bin 0 -> 278 bytes .../compose/ozonesecure-ha/keytabs/recon.keytab | Bin 0 -> 296 bytes .../main/compose/ozonesecure-ha/keytabs/s3g.keytab | Bin 0 -> 434 bytes .../main/compose/ozonesecure-ha/keytabs/scm.keytab | Bin 0 -> 586 bytes .../compose/ozonesecure-ha/keytabs/testuser.keytab | Bin 0 -> 152 bytes .../ozonesecure-ha/keytabs/testuser2.keytab | Bin 0 -> 154 bytes .../docker-krb5 => ozonesecure-ha}/krb5.conf | 4 +- .../dist/src/main/compose/ozonesecure-mr/.env | 1 + .../dist/src/main/compose/ozonesecure-mr/README.md | 2 +- .../compose/ozonesecure-mr/docker-compose.yaml | 30 +- .../src/main/compose/ozonesecure-mr/docker-config | 25 +- .../compose/ozonesecure-mr/keytabs/HTTP.keytab | Bin 0 -> 144 bytes .../main/compose/ozonesecure-mr/keytabs/dn.keytab | Bin 0 -> 278 bytes .../main/compose/ozonesecure-mr/keytabs/om.keytab | Bin 0 -> 278 bytes .../compose/ozonesecure-mr/keytabs/recon.keytab | Bin 0 -> 296 bytes .../main/compose/ozonesecure-mr/keytabs/s3g.keytab | Bin 0 -> 434 bytes .../main/compose/ozonesecure-mr/keytabs/scm.keytab | Bin 0 -> 586 bytes .../compose/ozonesecure-mr/keytabs/testuser.keytab | Bin 0 -> 152 bytes .../ozonesecure-mr/keytabs/testuser2.keytab | Bin 0 -> 154 bytes .../docker-krb5 => ozonesecure-mr}/krb5.conf | 6 +- .../dist/src/main/compose/ozonesecure/.env | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 23 +- .../src/main/compose/ozonesecure/docker-config | 24 +- .../main/compose/ozonesecure/keytabs/HTTP.keytab | Bin 0 -> 144 bytes .../src/main/compose/ozonesecure/keytabs/dn.keytab | Bin 0 -> 278 bytes .../src/main/compose/ozonesecure/keytabs/om.keytab | Bin 0 -> 278 bytes .../main/compose/ozonesecure/keytabs/recon.keytab | Bin 0 -> 296 bytes .../main/compose/ozonesecure/keytabs/s3g.keytab | Bin 0 -> 434 bytes .../main/compose/ozonesecure/keytabs/scm.keytab | Bin 0 -> 586 bytes .../compose/ozonesecure/keytabs/testuser.keytab | Bin 0 -> 152 bytes .../compose/ozonesecure/keytabs/testuser2.keytab | Bin 0 -> 154 bytes .../docker-krb5 => ozonesecure}/krb5.conf | 4 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +- .../definitions/ozone/definitions/persistence.yaml | 4 +- .../examples/ozone-ha/datanode-statefulset.yaml | 2 +- .../k8s/examples/ozone/datanode-statefulset.yaml | 2 +- hadoop-ozone/dist/src/main/keytabs/HTTP.keytab | Bin 0 -> 432 bytes hadoop-ozone/dist/src/main/keytabs/dn.keytab | Bin 0 -> 278 bytes hadoop-ozone/dist/src/main/keytabs/hadoop.keytab | Bin 0 -> 146 bytes hadoop-ozone/dist/src/main/keytabs/jhs.keytab | Bin 0 -> 142 bytes hadoop-ozone/dist/src/main/keytabs/nm.keytab | Bin 0 -> 138 bytes hadoop-ozone/dist/src/main/keytabs/om.keytab | Bin 0 -> 278 bytes hadoop-ozone/dist/src/main/keytabs/recon.keytab | Bin 0 -> 296 bytes hadoop-ozone/dist/src/main/keytabs/rm.keytab | Bin 0 -> 138 bytes hadoop-ozone/dist/src/main/keytabs/s3g.keytab | Bin 0 -> 434 bytes hadoop-ozone/dist/src/main/keytabs/scm.keytab | Bin 0 -> 586 bytes hadoop-ozone/dist/src/main/keytabs/testuser.keytab | Bin 0 -> 450 bytes .../dist/src/main/keytabs/testuser2.keytab | Bin 0 -> 154 bytes .../dist/src/main/keytabs/update-keytabs.sh | 77 +++++ hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 58 ++-- .../dist/src/main/license/update-jar-report.sh | 2 +- .../src/main/smoketest/admincli/container.robot | 11 +- .../dist/src/main/smoketest/basic/links.robot | 10 +- .../src/main/smoketest/basic/ozone-shell-lib.robot | 6 +- .../dist/src/main/smoketest/cli/classpath.robot | 14 +- .../dist/src/main/smoketest/commonlib.robot | 8 +- .../dist/src/main/smoketest/ozonefs/setup.robot | 10 +- .../main/smoketest/security/ozone-secure-fs.robot | 18 +- hadoop-ozone/dist/src/shell/ozone/ozone | 42 +-- .../fault-injection-test/mini-chaos-tests/pom.xml | 24 +- .../fault-injection-test/network-tests/pom.xml | 6 +- hadoop-ozone/fault-injection-test/pom.xml | 6 +- hadoop-ozone/insight/pom.xml | 42 +-- .../hadoop/ozone/insight/BaseInsightPoint.java | 4 +- hadoop-ozone/integration-test/pom.xml | 70 ++--- .../ozone/client/rpc/read/TestKeyInputStream.java | 63 +++- .../container/metrics/TestContainerMetrics.java | 2 +- .../server/TestSecureContainerServer.java | 2 +- .../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 31 +- .../TestDatanodeHddsVolumeFailureDetection.java | 6 + .../TestDatanodeHddsVolumeFailureToleration.java | 131 ++++++++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 99 ++++++ hadoop-ozone/interface-client/pom.xml | 10 +- hadoop-ozone/interface-storage/pom.xml | 30 +- hadoop-ozone/native-client/README.md | 4 +- hadoop-ozone/ozone-manager/pom.xml | 60 ++-- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 24 ++ .../S3MultipartUploadCompleteRequest.java | 3 +- hadoop-ozone/ozonefs-common/pom.xml | 42 +-- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 30 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 16 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 14 +- hadoop-ozone/ozonefs-shaded/pom.xml | 10 +- hadoop-ozone/ozonefs/pom.xml | 14 +- hadoop-ozone/pom.xml | 142 ++++----- hadoop-ozone/recon-codegen/pom.xml | 10 +- hadoop-ozone/recon/pom.xml | 38 +-- hadoop-ozone/s3gateway/pom.xml | 42 +-- hadoop-ozone/tools/pom.xml | 50 +-- .../ozone/debug/container/ContainerCommands.java | 2 +- .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 +- .../ozone/freon/ClosedContainerReplicator.java | 3 +- .../containergenerator/GeneratorDatanode.java | 2 +- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +- .../hadoop/ozone/audit/parser/TestAuditParser.java | 31 +- .../tools/src/test/resources/testaudit.log | 3 + .../resources/{testaudit.log => testloadaudit.log} | 24 +- pom.xml | 4 +- 224 files changed, 2973 insertions(+), 1208 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
