Attila Doroszlai created HDDS-14793:
---------------------------------------
Summary: Intermittent NPE in XceiverClientGrpc
Key: HDDS-14793
URL: https://issues.apache.org/jira/browse/HDDS-14793
Project: Apache Ozone
Issue Type: Bug
Components: Ozone Client
Reporter: Attila Doroszlai
{code}
Freon.Metadata-Generate :: Test freon ommg command
==============================================================================
[Setup] Create Volume and Buckets | PASS |
------------------------------------------------------------------------------
[Read] Bucket Information | PASS |
------------------------------------------------------------------------------
[Create] File in FILE_SYSTEM_OPTIMIZED Bucket | PASS |
------------------------------------------------------------------------------
[Read] File in FILE_SYSTEM_OPTIMIZED Bucket | FAIL |
255 != 0
{code}
{code}
Running command 'ozone freon ommg --operation READ_FILE -n 100 --volume volume1
--bucket bucket-fso --size 4096'
${rc} = 255
{code}
{code}
2026-03-09 04:58:43,337 [main] INFO freon.BaseFreonGenerator: Executing test
with prefix '' and number-of-tests 100
2026-03-09 04:58:43,391 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 % (0
out of 100)
2026-03-09 04:58:44,393 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 % (0
out of 100)
2026-03-09 04:58:45,064 [pool-2-thread-6] WARN impl.MetricsSystemImpl:
ozone-freon metrics system already initialized!
2026-03-09 04:58:45,394 [Thread-5] INFO freon.ProgressBar: Progress: 0.00 % (0
out of 100)
2026-03-09 04:58:45,458 [pool-2-thread-4] ERROR freon.BaseFreonGenerator: Error
on executing task 3
java.lang.NullPointerException: Cannot invoke
"org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc$XceiverClientProtocolServiceStub.withDeadlineAfter(long,
java.util.concurrent.TimeUnit)" because the return value of
"java.util.Map.get(Object)" is null
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandAsync(XceiverClientGrpc.java:701)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:513)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$5(XceiverClientGrpc.java:428)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:205)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:195)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:420)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:401)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:235)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getBlock$0(ContainerProtocolCalls.java:201)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.tryEachDatanode(ContainerProtocolCalls.java:155)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:200)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockDataUsingClient(BlockInputStream.java:239)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockData(BlockInputStream.java:222)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:154)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.readWithStrategy(BlockInputStream.java:300)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
at
org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:51)
at
org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:107)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
at
org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
at java.base/java.io.InputStream.read(InputStream.java:220)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.performReadOperation(OmMetadataGenerator.java:437)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.lambda$applyOperation$12(OmMetadataGenerator.java:340)
at com.codahale.metrics.Timer.time(Timer.java:101)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.applyOperation(OmMetadataGenerator.java:340)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.tryNextTask(BaseFreonGenerator.java:213)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.taskLoop(BaseFreonGenerator.java:194)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.lambda$startTaskRunners$0(BaseFreonGenerator.java:168)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
2026-03-09 04:58:45,459 [pool-2-thread-2] ERROR freon.BaseFreonGenerator: Error
on executing task 1
java.lang.NullPointerException: Cannot invoke
"org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc$XceiverClientProtocolServiceStub.withDeadlineAfter(long,
java.util.concurrent.TimeUnit)" because the return value of
"java.util.Map.get(Object)" is null
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandAsync(XceiverClientGrpc.java:701)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithRetry(XceiverClientGrpc.java:513)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.lambda$sendCommandWithTraceIDAndRetry$5(XceiverClientGrpc.java:428)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInSpan(TracingUtil.java:205)
at
org.apache.hadoop.hdds.tracing.TracingUtil.executeInNewSpan(TracingUtil.java:195)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommandWithTraceIDAndRetry(XceiverClientGrpc.java:420)
at
org.apache.hadoop.hdds.scm.XceiverClientGrpc.sendCommand(XceiverClientGrpc.java:401)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:235)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.lambda$getBlock$0(ContainerProtocolCalls.java:201)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.tryEachDatanode(ContainerProtocolCalls.java:155)
at
org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls.getBlock(ContainerProtocolCalls.java:200)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockDataUsingClient(BlockInputStream.java:239)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.getBlockData(BlockInputStream.java:222)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.initialize(BlockInputStream.java:154)
at
org.apache.hadoop.hdds.scm.storage.BlockInputStream.readWithStrategy(BlockInputStream.java:300)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
at
org.apache.hadoop.hdds.scm.storage.ByteArrayReader.readFromBlock(ByteArrayReader.java:51)
at
org.apache.hadoop.hdds.scm.storage.MultipartInputStream.readWithStrategy(MultipartInputStream.java:107)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:73)
at
org.apache.hadoop.hdds.scm.storage.ExtendedInputStream.read(ExtendedInputStream.java:61)
at
org.apache.hadoop.ozone.client.io.OzoneInputStream.read(OzoneInputStream.java:56)
at java.base/java.io.InputStream.read(InputStream.java:220)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.performReadOperation(OmMetadataGenerator.java:437)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.lambda$applyOperation$12(OmMetadataGenerator.java:340)
at com.codahale.metrics.Timer.time(Timer.java:101)
at
org.apache.hadoop.ozone.freon.OmMetadataGenerator.applyOperation(OmMetadataGenerator.java:340)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.tryNextTask(BaseFreonGenerator.java:213)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.taskLoop(BaseFreonGenerator.java:194)
at
org.apache.hadoop.ozone.freon.BaseFreonGenerator.lambda$startTaskRunners$0(BaseFreonGenerator.java:168)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
2026-03-09 04:58:46,395 [Thread-5] INFO freon.ProgressBar: Progress: 10.00 %
(10 out of 100)
One or more freon test is failed.
2026-03-09 04:58:46,613 [shutdown-hook-0] INFO metrics: type=TIMER,
name=READ_FILE, count=10, min=565.872325, max=897.601353,
mean=830.9511569000001, stddev=132.449695984493, median=896.9616,
p75=897.301304, p95=897.601353, p98=897.601353, p99=897.601353,
p999=897.601353, mean_rate=5.8031642117303175, m1=0.0, m5=0.0, m15=0.0,
rate_unit=events/second, duration_unit=milliseconds
2026-03-09 04:58:46,616 [shutdown-hook-0] INFO freon.BaseFreonGenerator: Total
execution time (sec): 3
2026-03-09 04:58:46,616 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
Failures: 2
2026-03-09 04:58:46,617 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
Successful executions: 8
2026-03-09 04:58:46,617 [shutdown-hook-0] INFO freon.BaseFreonGenerator:
Expected 100 --number-of-tests objects!, successfully executed 8
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
