[
https://issues.apache.org/jira/browse/HDDS-15423?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Tak-Lon (Stephen) Wu updated HDDS-15423:
----------------------------------------
Attachment:
hbase-cmf-HBASE-1-REGIONSERVER-taklwu-test-2.vpc.cloudera.com.log.out1
> Ozone Client CheckSum failed during stream read
> -----------------------------------------------
>
> Key: HDDS-15423
> URL: https://issues.apache.org/jira/browse/HDDS-15423
> Project: Apache Ozone
> Issue Type: Bug
> Components: Ozone Client
> Reporter: Tak-Lon (Stephen) Wu
> Priority: Major
> Attachments:
> hbase-cmf-HBASE-1-REGIONSERVER-taklwu-test-2.vpc.cloudera.com.log.out1
>
>
> Another test with HBase with Ozone, during the YCSB, stream read was enabled
> and many checksum WARNs/ERRORs were found while reading data block or
> bloomfilter block.
> {code}
> ozone.client.stream.readblock.enable = true
> hdds.container.ratis.datastream.enabled = true
> ozone.client.verify.checksum = true
> hdds.grpc.tls.enabled = false
>
> {code}
> it may be related to the HDDS-15422 that reverse scan close the stream early,
> but we found a lot of check sum error when reading datablock or bloomfilter
> block.
> {code}
> 2026-05-29 10:08:48,022 WARN
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream:
> [RpcServer.default.FPBQ.Fifo.handler=13,queue=4,port=22101]:
> stream5774-reader889645: Failed to setCompleted: already completed
> exceptionally
> java.util.concurrent.ExecutionException:
> org.apache.hadoop.ozone.common.OzoneChecksumException: Checksum mismatched:
> this.checksums(0) != that.checksums(0), thisStartIndex=0,
> this=ChecksumData{CRC32, bytesPerChecksum=1048576, checksums=[0D A7 20 21]},
> that=ChecksumData{CRC32, bytesPerChecksum=1048576, checksums=[CA 46 66 9D]}
> at
> java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396)
> ~[?:?]
> at
> java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream$StreamingReader.setCompleted(StreamBlockInputStream.java:550)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream$StreamingReader.onCompleted(StreamBlockInputStream.java:522)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream.closeStream(StreamBlockInputStream.java:243)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream.seek(StreamBlockInputStream.java:212)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.seek(MultipartInputStream.java:177)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readFully(MultipartInputStream.java:207)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.fs.ozone.OzoneFSInputStream.read(OzoneFSInputStream.java:174)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:260)
> ~[hadoop-common-3.4.2.7.3.2.10000-238.jar:?]
> at jdk.internal.reflect.GeneratedMethodAccessor393.invoke(Unknown
> Source) ~[?:?]
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:?]
> at java.lang.reflect.Method.invoke(Method.java:569) ~[?:?]
> at
> org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtraDirectly(BlockIOUtils.java:355)
> ~[hbase-common-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.util.BlockIOUtils.preadWithExtra(BlockIOUtils.java:305)
> ~[hbase-common-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readAtOffset(HFileBlock.java:1501)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1801)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1539)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1388)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(HFileReaderImpl.java:1306)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.NoOpIndexBlockEncoder$NoOpEncodedSeeker.loadDataBlockWithScanInfo(NoOpIndexBlockEncoder.java:340)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$CellBasedKeyBlockIndexReaderV2.loadDataBlockWithScanInfo(HFileBlockIndex.java:508)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:683)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.seekTo(HFileReaderImpl.java:636)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:322)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:230)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.StoreScanner.seekScanners(StoreScanner.java:425)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:269)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1726)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1715)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.RegionScannerImpl.initializeScanners(RegionScannerImpl.java:171)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.RegionScannerImpl.<init>(RegionScannerImpl.java:147)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:3174)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.lambda$getScanner$5(HRegion.java:3159)
> ~[hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
> [hbase-common-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3145)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3140)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:3134)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2680)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2605)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at
> org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45432)
> [hbase-protocol-shaded-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85)
> [hbase-server-2.6.3.7.3.2.10000-238.jar:2.6.3.7.3.2.10000-238]
> Caused by: org.apache.hadoop.ozone.common.OzoneChecksumException: Checksum
> mismatched: this.checksums(0) != that.checksums(0), thisStartIndex=0,
> this=ChecksumData{CRC32, bytesPerChecksum=1048576, checksums=[0D A7 20 21]},
> that=ChecksumData{CRC32, bytesPerChecksum=1048576, checksums=[CA 46 66 9D]}
> at
> org.apache.hadoop.ozone.common.ChecksumData.verifyChecksumDataMatches(ChecksumData.java:127)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.ozone.common.Checksum.verifyChecksum(Checksum.java:315)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.ozone.common.Checksum.verifyChecksum(Checksum.java:291)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream$StreamingReader.onNext(StreamBlockInputStream.java:490)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.hadoop.hdds.scm.storage.StreamBlockInputStream$StreamingReader.onNext(StreamBlockInputStream.java:388)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.ozone.shaded.org.apache.ratis.thirdparty.io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onMessage(ClientCalls.java:551)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.ozone.shaded.org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInternal(ClientCallImpl.java:661)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.ozone.shaded.org.apache.ratis.thirdparty.io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1MessagesAvailable.runInContext(ClientCallImpl.java:648)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.ozone.shaded.org.apache.ratis.thirdparty.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> org.apache.ozone.shaded.org.apache.ratis.thirdparty.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
> ~[ozone-filesystem-hadoop3-2.2.0.7.3.2.10000-238.jar:?]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> ~[?:?]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> ~[?:?]
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
