This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-7593 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 44b12428c2bbd150a41f64d34ad5c466ceb6d26e Merge: 825c340bae 97038ef097 Author: Siyao Meng <[email protected]> AuthorDate: Thu Mar 21 17:50:35 2024 -0700 Merge remote-tracking branch 'asf/master' into HDDS-7593 Conflicts: hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java .github/workflows/ci.yml | 16 +- .github/workflows/intermittent-test-check.yml | 6 +- .github/workflows/populate-cache.yml | 74 +++ .github/workflows/repeat-acceptance.yml | 12 +- .gitignore | 35 +- dev-support/rat/rat-exclusions.txt | 73 +++ .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 + .../hadoop/hdds/scm/XceiverClientMetrics.java | 44 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 27 +- .../ozone/client/io/BlockInputStreamFactory.java | 6 +- .../client/io/BlockInputStreamFactoryImpl.java | 15 +- .../hadoop/ozone/client/io/ECBlockInputStream.java | 16 +- .../ozone/client/io/ECBlockInputStreamFactory.java | 6 +- .../client/io/ECBlockInputStreamFactoryImpl.java | 13 +- .../ozone/client/io/ECBlockInputStreamProxy.java | 14 +- .../io/ECBlockReconstructedStripeInputStream.java | 10 +- .../hdds/scm/storage/DummyBlockInputStream.java | 9 +- .../storage/DummyBlockInputStreamWithRetry.java | 9 +- .../hdds/scm/storage/TestBlockInputStream.java | 24 +- .../hadoop/ozone/client/io/ECStreamTestUtil.java | 6 +- .../client/io/TestBlockInputStreamFactoryImpl.java | 14 +- .../ozone/client/io/TestECBlockInputStream.java | 110 +++- .../client/io/TestECBlockInputStreamProxy.java | 13 +- .../io/TestECBlockReconstructedInputStream.java | 10 +- .../TestECBlockReconstructedStripeInputStream.java | 12 +- hadoop-hdds/common/pom.xml | 5 - .../apache/hadoop/hdds/fs/SpaceUsageSource.java | 4 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 88 +-- .../hadoop/hdds/scm/container/ContainerInfo.java | 42 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 8 + .../java/org/apache/hadoop/hdds/utils/IOUtils.java | 4 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../hadoop/util/PerformanceMetricsInitializer.java | 2 +- .../common/src/main/resources/ozone-default.xml | 9 +- .../hadoop/hdds/fs/MockSpaceUsageSource.java | 22 + .../container/common/helpers/ContainerMetrics.java | 19 + .../container/common/impl/HddsDispatcher.java | 77 ++- .../container/common/volume/StorageVolume.java | 5 - .../ozone/container/common/volume/VolumeInfo.java | 93 +-- .../container/common/volume/VolumeInfoMetrics.java | 6 +- .../ozone/container/common/volume/VolumeUsage.java | 125 ++-- .../ECReconstructionCoordinator.java | 7 +- .../container/keyvalue/KeyValueContainerCheck.java | 6 +- .../ozone/container/keyvalue/KeyValueHandler.java | 2 + .../ozoneimpl/ContainerScannerConfiguration.java | 4 +- .../container/common/impl/TestHddsDispatcher.java | 6 +- .../common/volume/TestReservedVolumeSpace.java | 28 +- .../TestContainerScannerConfiguration.java | 4 +- hadoop-hdds/docs/.gitignore | 2 + hadoop-hdds/docs/content/interface/HttpFS.md | 2 +- hadoop-hdds/docs/content/interface/ReconApi.zh.md | 600 +++++++++++++++++- hadoop-hdds/docs/pom.xml | 24 - hadoop-hdds/framework/pom.xml | 2 + hadoop-hdds/hadoop-dependency-test/pom.xml | 5 + hadoop-hdds/pom.xml | 39 -- hadoop-hdds/rocks-native/pom.xml | 3 +- .../org/apache/ozone/graph/TestPrintableGraph.java | 84 +++ .../hadoop/hdds/scm/ha/SCMRatisResponse.java | 20 +- .../hdds/scm/server/SCMContainerMetrics.java | 1 - .../cli/datanode/DecommissionStatusSubCommand.java | 94 ++- .../client/checksum/ECBlockChecksumComputer.java | 50 +- .../checksum/ReplicatedBlockChecksumComputer.java | 23 +- .../hadoop/ozone/client/io/KeyInputStream.java | 34 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 25 +- .../ozone/client/io/TestKeyInputStreamEC.java | 12 +- .../java/org/apache/hadoop/ozone/OzoneAcl.java | 211 +++---- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 20 +- .../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 6 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 9 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 14 +- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 120 ++-- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 29 +- .../org/apache/hadoop/ozone/TestOzoneAcls.java | 154 ++--- .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 17 +- .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 2 +- .../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 8 +- .../hadoop/ozone/om/helpers/TestOzoneAclUtil.java | 38 +- hadoop-ozone/dev-support/checks/rat.sh | 8 +- hadoop-ozone/dist/.gitignore | 1 + hadoop-ozone/dist/src/main/license/jar-report.txt | 3 +- .../dist/src/main/smoketest/recon/recon-api.robot | 12 +- .../smoketest/security/ozone-secure-tenant.robot | 2 +- .../dist/src/main/smoketest/topology/cli.robot | 21 +- hadoop-ozone/dist/src/shell/ozone/ozone | 6 + .../fault-injection-test/mini-chaos-tests/pom.xml | 5 - .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 1 + hadoop-ozone/httpfsgateway/pom.xml | 4 - hadoop-ozone/integration-test/pom.xml | 5 - .../fs/contract/AbstractContractAppendTest.java | 2 + .../fs/contract/AbstractContractConcatTest.java | 2 + .../AbstractContractCopyFromLocalTest.java | 2 + .../AbstractContractGetFileStatusTest.java | 2 + .../AbstractContractMultipartUploaderTest.java | 4 + .../fs/contract/AbstractContractOpenTest.java | 2 + .../AbstractContractRootDirectoryTest.java | 2 + .../fs/contract/AbstractContractSeekTest.java | 4 + .../fs/contract/AbstractContractSetTimesTest.java | 2 + .../fs/contract/AbstractContractUnbufferTest.java | 2 + .../ozone/AbstractRootedOzoneFileSystemTest.java | 111 +++- .../org/apache/hadoop/fs/ozone/TestSafeMode.java | 2 +- .../hdds/scm/TestStorageContainerManager.java | 383 ++++++------ .../scm/TestStorageContainerManagerHelper.java | 198 ------ .../scm/node/TestDecommissionAndMaintenance.java | 2 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 + .../hadoop/ozone/MiniOzoneClusterProvider.java | 34 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 + .../org/apache/hadoop/ozone/TestBlockTokens.java | 7 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 29 + .../hadoop/ozone/TestSecureOzoneCluster.java | 34 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 66 +- .../rpc/TestOzoneRpcClientForAclAuditLog.java | 4 +- .../apache/hadoop/ozone/om/TestBucketOwner.java | 6 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 29 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 557 +++++++++-------- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 67 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 18 +- .../ozone/om/multitenant/RangerUserRequest.java | 28 +- .../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 +- .../hadoop/ozone/shell/TestOzoneTenantShell.java | 68 +- .../tools/contract/AbstractContractDistCpTest.java | 2 + .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 14 +- .../hadoop/ozone/om/helpers/OzoneAclStorage.java | 15 +- .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 9 +- .../ozone/om/helpers/TestOmPrefixInfoCodec.java | 2 +- hadoop-ozone/ozone-manager/pom.xml | 5 - .../hadoop/ozone/om/BucketUtilizationMetrics.java | 118 ++++ .../org/apache/hadoop/ozone/om/ListIterator.java | 2 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 34 - .../org/apache/hadoop/ozone/om/OzoneManager.java | 23 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 5 +- .../RangerRestMultiTenantAccessController.java | 681 --------------------- .../ozone/om/ratis/OzoneManagerStateMachine.java | 16 +- .../om/request/bucket/OMBucketCreateRequest.java | 3 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +- .../ozone/om/TestBucketUtilizationMetrics.java | 126 ++++ .../apache/hadoop/ozone/om/TestChunkStreams.java | 9 +- .../om/ratis/TestOzoneManagerStateMachine.java | 7 +- .../request/TestOMClientRequestWithUserInfo.java | 92 ++- .../om/request/file/TestOMFileCreateRequest.java | 6 +- .../om/request/key/TestOMKeyCreateRequest.java | 2 +- .../TestS3InitiateMultipartUploadRequest.java | 11 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +- .../key/acl/prefix/TestOMPrefixAclResponse.java | 4 +- .../ozone/om/snapshot/TestSnapshotDiffManager.java | 11 +- .../security/acl/TestOzoneNativeAuthorizer.java | 18 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 6 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 1 + hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 2 + hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 + hadoop-ozone/ozonefs-shaded/pom.xml | 1 + hadoop-ozone/pom.xml | 67 +- hadoop-ozone/recon/.gitignore | 1 + .../webapps/recon/ozone-recon-web/api/db.json | 67 ++ .../webapps/recon/ozone-recon-web/api/routes.json | 3 +- .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 15 +- .../ozone-recon-web/src/views/insights/om/om.tsx | 120 +++- .../hadoop/ozone/recon/api/TestEndpoints.java | 12 +- .../hadoop/ozone/recon/common/CommonUtils.java | 5 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 19 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +- .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 66 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 3 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 2 +- .../hadoop/ozone/freon/DatanodeSimulator.java | 4 +- .../freon/containergenerator/GeneratorOm.java | 4 +- .../apache/hadoop/ozone/repair/OzoneRepair.java | 64 ++ .../org/apache/hadoop/ozone/repair/RDBRepair.java | 58 ++ .../hadoop/ozone/repair/om/SnapshotRepair.java | 200 ++++++ .../hadoop/ozone/repair/om/package-info.java | 22 + .../apache/hadoop/ozone/repair/package-info.java | 22 + .../ozone/shell/tenant/GetUserInfoHandler.java | 44 +- .../shell/tenant/TenantAssignAdminHandler.java | 21 +- .../ozone/shell/tenant/TenantCreateHandler.java | 15 +- .../ozone/shell/tenant/TenantDeleteHandler.java | 17 +- .../ozone/shell/tenant/TenantListHandler.java | 31 +- .../ozone/shell/tenant/TenantListUsersHandler.java | 19 +- .../shell/tenant/TenantRevokeAdminHandler.java | 21 +- pom.xml | 58 +- 181 files changed, 3879 insertions(+), 2948 deletions(-) diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 2c30abd3d5,b62415395d..b66999de1f --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@@ -113,30 -112,31 +114,36 @@@ public class BlockInputStream extends B private final Function<BlockID, BlockLocationInfo> refreshFunction; - public BlockInputStream(BlockLocationInfo blockInfo, Pipeline pipeline, - Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum, - public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline, ++ public BlockInputStream( ++ BlockLocationInfo blockInfo, ++ Pipeline pipeline, + Token<OzoneBlockTokenIdentifier> token, XceiverClientFactory xceiverClientFactory, - Function<BlockID, BlockLocationInfo> refreshFunction) { + Function<BlockID, BlockLocationInfo> refreshFunction, + OzoneClientConfig config) { - this.blockID = blockId; - this.length = blockLen; + this.blockInfo = blockInfo; + this.blockID = blockInfo.getBlockID(); + this.length = blockInfo.getLength(); setPipeline(pipeline); tokenRef.set(token); - this.verifyChecksum = verifyChecksum; + this.verifyChecksum = config.isChecksumVerify(); this.xceiverClientFactory = xceiverClientFactory; this.refreshFunction = refreshFunction; + this.retryPolicy = + HddsClientUtils.createRetryPolicy(config.getMaxReadRetryCount(), + TimeUnit.SECONDS.toMillis(config.getReadRetryInterval())); } + // only for unit tests public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline, Token<OzoneBlockTokenIdentifier> token, - boolean verifyChecksum, - XceiverClientFactory xceiverClientFactory) { + XceiverClientFactory xceiverClientFactory, + OzoneClientConfig config + ) { - this(blockId, blockLen, pipeline, token, - xceiverClientFactory, null, config); + this(new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockId).setLength(blockLen)), - pipeline, token, verifyChecksum, - xceiverClientFactory, null); ++ pipeline, token, xceiverClientFactory, null, config); } + /** * Initialize the BlockInputStream. Get the BlockData (list of chunks) from * the Container and create the ChunkInputStreams for each Chunk in the Block. diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java index b9233f42d5,6bcdc3c481..7edc498cf6 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java @@@ -76,16 -77,18 +77,18 @@@ public class BlockInputStreamFactoryImp */ public BlockExtendedInputStream create(ReplicationConfig repConfig, BlockLocationInfo blockInfo, Pipeline pipeline, - Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum, + Token<OzoneBlockTokenIdentifier> token, XceiverClientFactory xceiverFactory, - Function<BlockID, BlockLocationInfo> refreshFunction) { + Function<BlockID, BlockLocationInfo> refreshFunction, + OzoneClientConfig config) { if (repConfig.getReplicationType().equals(HddsProtos.ReplicationType.EC)) { return new ECBlockInputStreamProxy((ECReplicationConfig)repConfig, - blockInfo, verifyChecksum, xceiverFactory, refreshFunction, - ecBlockStreamFactory); + blockInfo, xceiverFactory, refreshFunction, + ecBlockStreamFactory, config); } else { - return new BlockInputStream(blockInfo, pipeline, token, verifyChecksum, xceiverFactory, - refreshFunction); - return new BlockInputStream(blockInfo.getBlockID(), blockInfo.getLength(), ++ return new BlockInputStream(blockInfo, + pipeline, token, xceiverFactory, refreshFunction, + config); } } diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java index ca3199d8ac,a89097533d..2987a9b613 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java @@@ -23,8 -23,8 +23,9 @@@ import java.util.Map import java.util.function.Function; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; + import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier; @@@ -48,11 -48,10 +49,11 @@@ class DummyBlockInputStream extends Blo XceiverClientFactory xceiverClientManager, Function<BlockID, BlockLocationInfo> refreshFunction, List<ChunkInfo> chunkList, - Map<String, byte[]> chunks) { + Map<String, byte[]> chunks, + OzoneClientConfig config) { - super(blockId, blockLen, pipeline, token, + super(new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockId).setLength(blockLen)), - pipeline, token, verifyChecksum, - xceiverClientManager, refreshFunction); ++ pipeline, token, + xceiverClientManager, refreshFunction, config); this.chunkDataMap = chunks; this.chunks = chunkList; diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java index d66c76dcdd,6d12614228..172e62887b --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java @@@ -23,8 -23,8 +23,9 @@@ import java.util.Map import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; + import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; diff --cc hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java index f755817816,21b088ce85..0012d691f9 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java @@@ -22,9 -22,10 +22,11 @@@ import com.google.common.primitives.Byt import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.ContainerBlockID; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; + import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; @@@ -408,12 -411,14 +419,15 @@@ public class TestBlockInputStream .thenReturn(blockLocationInfo); when(blockLocationInfo.getPipeline()).thenReturn(newPipeline); + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setChecksumVerify(false); - BlockInputStream subject = new BlockInputStream(blockID, blockSize, + BlockInputStream subject = new BlockInputStream( + new BlockLocationInfo(new BlockLocationInfo.Builder().setBlockID(blockID).setLength(blockSize)), - pipeline, null, false, clientFactory, refreshFunction) { + pipeline, null, clientFactory, refreshFunction, + clientConfig) { @Override - protected List<ChunkInfo> getChunkInfoListUsingClient() { - return chunks; + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + return stream; } @Override diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java index 6b6be1abd4,2d40841ee4..15e84cf370 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java @@@ -59,13 -59,11 +60,13 @@@ public class KeyInputStream extends Mul OmKeyInfo keyInfo, List<OmKeyLocationInfo> blockInfos, XceiverClientFactory xceiverClientFactory, - boolean verifyChecksum, Function<OmKeyInfo, OmKeyInfo> retryFunction, - BlockInputStreamFactory blockStreamFactory) { + BlockInputStreamFactory blockStreamFactory, + OzoneClientConfig config) { + boolean isHsyncFile = keyInfo.getMetadata().containsKey(OzoneConsts.HSYNC_CLIENT_ID); List<BlockExtendedInputStream> partStreams = new ArrayList<>(); - for (OmKeyLocationInfo omKeyLocationInfo : blockInfos) { + for (int i = 0; i < blockInfos.size(); i++) { + OmKeyLocationInfo omKeyLocationInfo = blockInfos.get(i); if (LOG.isDebugEnabled()) { LOG.debug("Adding stream for accessing {}. The stream will be " + "initialized later.", omKeyLocationInfo); diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 85e7c2a76e,47f5314bd6..845140293a --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@@ -304,23 -357,24 +357,24 @@@ public class TestOmMetrics RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); doKeyOps(keyArgs); - MetricsRecordBuilder omMetrics = getMetrics("OMMetrics"); - assertCounter("NumKeyOps", 8L, omMetrics); - assertCounter("NumKeyAllocate", 1L, omMetrics); - assertCounter("NumKeyLookup", 1L, omMetrics); - assertCounter("NumKeyDeletes", 1L, omMetrics); - assertCounter("NumKeyLists", 1L, omMetrics); - assertCounter("NumTrashKeyLists", 1L, omMetrics); - assertCounter("NumKeys", 0L, omMetrics); - assertCounter("NumInitiateMultipartUploads", 1L, omMetrics); - assertCounter("NumListOpenFiles", 1L, omMetrics); + omMetrics = getMetrics("OMMetrics"); + - assertEquals(initialNumKeyOps + 7, getLongCounter("NumKeyOps", omMetrics)); ++ assertEquals(initialNumKeyOps + 8, getLongCounter("NumKeyOps", omMetrics)); + assertEquals(initialNumKeyAllocate + 1, getLongCounter("NumKeyAllocate", omMetrics)); + assertEquals(initialNumKeyLookup + 1, getLongCounter("NumKeyLookup", omMetrics)); + assertEquals(initialNumKeyDeletes + 1, getLongCounter("NumKeyDeletes", omMetrics)); + assertEquals(initialNumKeyLists + 1, getLongCounter("NumKeyLists", omMetrics)); + assertEquals(initialNumTrashKeyLists + 1, getLongCounter("NumTrashKeyLists", omMetrics)); - assertEquals(initialNumKeys, getLongCounter("NumKeys", omMetrics)); ++ assertEquals(initialNumKeys + 1, getLongCounter("NumKeys", omMetrics)); + assertEquals(initialNumInitiateMultipartUploads + 1, getLongCounter("NumInitiateMultipartUploads", omMetrics)); keyArgs = createKeyArgs(volumeName, bucketName, new ECReplicationConfig("rs-3-2-1024K")); doKeyOps(keyArgs); + omMetrics = getMetrics("OMMetrics"); - assertCounter("NumKeyOps", 16L, omMetrics); - assertCounter("EcKeyCreateTotal", 1L, omMetrics); - assertEquals(initialNumKeys, getLongCounter("NumKeys", omMetrics)); ++ assertEquals(initialNumKeys + 2, getLongCounter("NumKeys", omMetrics)); + assertEquals(initialEcKeyCreateTotal + 1, getLongCounter("EcKeyCreateTotal", omMetrics)); keyArgs = createKeyArgs(volumeName, bucketName, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE)); @@@ -371,21 -425,22 +425,22 @@@ doKeyOps(keyArgs); omMetrics = getMetrics("OMMetrics"); - assertCounter("NumKeyOps", 31L, omMetrics); - assertCounter("NumKeyAllocate", 6L, omMetrics); - assertCounter("NumKeyLookup", 3L, omMetrics); - assertCounter("NumKeyDeletes", 4L, omMetrics); - assertCounter("NumKeyLists", 3L, omMetrics); - assertCounter("NumTrashKeyLists", 3L, omMetrics); - assertCounter("NumInitiateMultipartUploads", 3L, omMetrics); - - assertCounter("NumKeyAllocateFails", 1L, omMetrics); - assertCounter("NumKeyLookupFails", 1L, omMetrics); - assertCounter("NumKeyDeleteFails", 1L, omMetrics); - assertCounter("NumKeyListFails", 1L, omMetrics); - assertCounter("NumTrashKeyListFails", 1L, omMetrics); - assertCounter("NumInitiateMultipartUploadFails", 1L, omMetrics); - assertCounter("NumKeys", 2L, omMetrics); - assertEquals(initialNumKeyOps + 28, getLongCounter("NumKeyOps", omMetrics)); ++ assertEquals(initialNumKeyOps + 31, getLongCounter("NumKeyOps", omMetrics)); + assertEquals(initialNumKeyAllocate + 6, getLongCounter("NumKeyAllocate", omMetrics)); + assertEquals(initialNumKeyLookup + 3, getLongCounter("NumKeyLookup", omMetrics)); + assertEquals(initialNumKeyDeletes + 4, getLongCounter("NumKeyDeletes", omMetrics)); + assertEquals(initialNumKeyLists + 3, getLongCounter("NumKeyLists", omMetrics)); + assertEquals(initialNumTrashKeyLists + 3, getLongCounter("NumTrashKeyLists", omMetrics)); + assertEquals(initialNumInitiateMultipartUploads + 3, getLongCounter("NumInitiateMultipartUploads", omMetrics)); + + assertEquals(initialNumKeyAllocateFails + 1, getLongCounter("NumKeyAllocateFails", omMetrics)); + assertEquals(initialNumKeyLookupFails + 1, getLongCounter("NumKeyLookupFails", omMetrics)); + assertEquals(initialNumKeyDeleteFails + 1, getLongCounter("NumKeyDeleteFails", omMetrics)); + assertEquals(initialNumKeyListFails + 1, getLongCounter("NumKeyListFails", omMetrics)); + assertEquals(initialNumTrashKeyListFails + 1, getLongCounter("NumTrashKeyListFails", omMetrics)); + assertEquals(initialNumInitiateMultipartUploadFails + 1, getLongCounter( + "NumInitiateMultipartUploadFails", omMetrics)); + assertEquals(initialNumKeys + 2, getLongCounter("NumKeys", omMetrics)); keyArgs = createKeyArgs(volumeName, bucketName, new ECReplicationConfig("rs-3-2-1024K")); diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java index df7f5b67b4,e0d5ef4084..33a33ad807 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java @@@ -90,7 -94,10 +94,10 @@@ public class TestChunkStreams } private BlockInputStream createStream(byte[] buf, int offset) { - return new BlockInputStream(null, 100L, null, null, true, null) { + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setChecksumVerify(true); - return new BlockInputStream(null, 100, null, null, null, ++ return new BlockInputStream(null, 100L, null, null, null, + clientConfig) { private long pos; private final ByteArrayInputStream in = new ByteArrayInputStream(buf, offset, 100); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
