[
https://issues.apache.org/jira/browse/HDDS-7197?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sammi Chen updated HDDS-7197:
-----------------------------
Description:
Currently RocksDB registers its statistics as jmx metrics with db directory
name, which is "container.db", same for all rocksdb . Then only the first
register will succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
Also, there is following logs with the current implementation.
2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
ManagedStatistics is not closed properly
was:
Currently RocksDB registers its statistics as jmx metrics with db directory
name, which is "container.db", same for all rocksdb . Then only the first
register will succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
ManagedStatistics is not closed properly
> RocksDB instance statitics register as metrics collision
> --------------------------------------------------------
>
> Key: HDDS-7197
> URL: https://issues.apache.org/jira/browse/HDDS-7197
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Sammi Chen
> Assignee: Sammi Chen
> Priority: Major
>
> Currently RocksDB registers its statistics as jmx metrics with db directory
> name, which is "container.db", same for all rocksdb . Then only the first
> register will succeed, others will fail with following logs.
>
> 2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
> name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
> org.apache.hadoop.metrics2.MetricsException:
> org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
> at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
> at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
> at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
> at
> org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
> at
> org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
> at
> org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
> at
> org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
> at
> org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
> at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
> at picocli.CommandLine.access$1300(CommandLine.java:145)
> at
> picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
> at
> picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
> at picocli.CommandLine.execute(CommandLine.java:2078)
> at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
> at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
> at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
> Caused by: org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
> ... 30 more
>
> Also, there is following logs with the current implementation.
> 2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
> ManagedStatistics is not closed properly
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
