[
https://issues.apache.org/jira/browse/HDDS-14793?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Attila Doroszlai reassigned HDDS-14793:
---------------------------------------
Assignee: Rishabh Patel
> Intermittent NPE in XceiverClientGrpc
> -------------------------------------
>
> Key: HDDS-14793
> URL: https://issues.apache.org/jira/browse/HDDS-14793
> Project: Apache Ozone
> Issue Type: Bug
> Components: Ozone Client
> Reporter: Attila Doroszlai
> Assignee: Rishabh Patel
> Priority: Major
>
> {code}
> Freon.Metadata-Generate :: Test freon ommg command
> ==============================================================================
> [Setup] Create Volume and Buckets | PASS |
> ------------------------------------------------------------------------------
> [Read] Bucket Information | PASS |
> ------------------------------------------------------------------------------
> [Create] File in FILE_SYSTEM_OPTIMIZED Bucket | PASS |
> ------------------------------------------------------------------------------
> [Read] File in FILE_SYSTEM_OPTIMIZED Bucket | FAIL |
> 255 != 0
> {code}
> {code}
> Running command 'ozone freon ommg --operation READ_FILE -n 100 --volume
> volume1 --bucket bucket-fso --size 4096'
> ${rc} = 255
> {code}
> {code}
> 2026-03-09 04:58:43,337 [main] INFO freon.BaseFreonGenerator: Executing test
> with prefix '' and number-of-tests 100
> 2026-03-09 04:58:43,391 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 %
> (0 out of 100)
> 2026-03-09 04:58:44,393 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 %
> (0 out of 100)
> 2026-03-09 04:58:45,064 [pool-2-thread-6] WARN impl.MetricsSystemImpl:
> ozone-freon metrics system already initialized!
> 2026-03-09 04:58:45,394 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 %
> (0 out of 100)
> 2026-03-09 04:58:45,458 [pool-2-thread-4] ERROR freon.BaseFreonGenerator:
> Error on executing task 3
> java.lang.NullPointerException: Cannot invoke
> "org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc$XceiverClientProtocolServiceStub.withDeadlineAfter(long,
> java.util.concurrent.TimeUnit)" because the return value of
> "java.util.Map.get(Object)" is null
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandAsync(XceiverClientGrpc.java:701)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:513)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$5(XceiverClientGrpc.java:428)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:205)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:195)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:420)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:401)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:235)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getBlock$0(ContainerProtocolCalls.java:201)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.tryEachDatanode(ContainerProtocolCalls.java:155)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:200)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockDataUsingClient(BlockInputStream.java:239)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockData(BlockInputStream.java:222)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:154)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.readWithStrategy(BlockInputStream.java:300)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
> at
> org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:51)
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:107)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
> at
> org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
> at java.base/java.io.InputStream.read(InputStream.java:220)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.performReadOperation(OmMetadataGenerator.java:437)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.lambda$applyOperation$12(OmMetadataGenerator.java:340)
> at com.codahale.metrics.Timer.time(Timer.java:101)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.applyOperation(OmMetadataGenerator.java:340)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.tryNextTask(BaseFreonGenerator.java:213)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.taskLoop(BaseFreonGenerator.java:194)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.lambda$startTaskRunners$0(BaseFreonGenerator.java:168)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> 2026-03-09 04:58:45,459 [pool-2-thread-2] ERROR freon.BaseFreonGenerator:
> Error on executing task 1
> java.lang.NullPointerException: Cannot invoke
> "org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc$XceiverClientProtocolServiceStub.withDeadlineAfter(long,
> java.util.concurrent.TimeUnit)" because the return value of
> "java.util.Map.get(Object)" is null
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandAsync(XceiverClientGrpc.java:701)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:513)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$5(XceiverClientGrpc.java:428)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:205)
> at
> org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:195)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:420)
> at
> org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:401)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:235)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getBlock$0(ContainerProtocolCalls.java:201)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.tryEachDatanode(ContainerProtocolCalls.java:155)
> at
> org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:200)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockDataUsingClient(BlockInputStream.java:239)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockData(BlockInputStream.java:222)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:154)
> at
> org.apache.hadoop.hdds.scm.storage.BlockInputStream.readWithStrategy(BlockInputStream.java:300)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
> at
> org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:51)
> at
> org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:107)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
> at
> org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
> at
> org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
> at java.base/java.io.InputStream.read(InputStream.java:220)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.performReadOperation(OmMetadataGenerator.java:437)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.lambda$applyOperation$12(OmMetadataGenerator.java:340)
> at com.codahale.metrics.Timer.time(Timer.java:101)
> at
> org.apache.hadoop.ozone.freon.OmMetadataGenerator.applyOperation(OmMetadataGenerator.java:340)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.tryNextTask(BaseFreonGenerator.java:213)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.taskLoop(BaseFreonGenerator.java:194)
> at
> org.apache.hadoop.ozone.freon.BaseFreonGenerator.lambda$startTaskRunners$0(BaseFreonGenerator.java:168)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
> at java.base/java.lang.Thread.run(Thread.java:1583)
> 2026-03-09 04:58:46,395 [Thread-5] INFO freon.ProgressBar: Progress: 10.00 %
> (10 out of 100)
> One or more freon test is failed.
> 2026-03-09 04:58:46,613 [shutdown-hook-0] INFO metrics: type=TIMER,
> name=READ_FILE, count=10, min=565.872325, max=897.601353,
> mean=830.9511569000001, stddev=132.449695984493, median=896.9616,
> p75=897.301304, p95=897.601353, p98=897.601353, p99=897.601353,
> p999=897.601353, mean_rate=5.8031642117303175, m1=0.0, m5=0.0, m15=0.0,
> rate_unit=events/second, duration_unit=milliseconds
> 2026-03-09 04:58:46,616 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
> Total execution time (sec): 3
> 2026-03-09 04:58:46,616 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
> Failures: 2
> 2026-03-09 04:58:46,617 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
> Successful executions: 8
> 2026-03-09 04:58:46,617 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
> Expected 100 --number-of-tests objects!, successfully executed 8
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
