[
https://issues.apache.org/jira/browse/HDDS-7197?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sammi Chen updated HDDS-7197:
-----------------------------
Description:
Currently RocksDB registers its statistics as jmx metrics with db directory
name, which is "container.db", same for all rocksdb . Then only the first
register will succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
ManagedStatistics is not closed properly
was:
Currently RocksDB registers its statistics as jmx metrics with db file name,
which is same for all V3 and V2 containers. Then only the first register will
succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
ManagedStatistics is not closed properly
> RocksDB instance statitics register as metrics collision
> --------------------------------------------------------
>
> Key: HDDS-7197
> URL: https://issues.apache.org/jira/browse/HDDS-7197
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Sammi Chen
> Assignee: Sammi Chen
> Priority: Major
>
> Currently RocksDB registers its statistics as jmx metrics with db directory
> name, which is "container.db", same for all rocksdb . Then only the first
> register will succeed, others will fail with following logs.
>
> 2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
> name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
> org.apache.hadoop.metrics2.MetricsException:
> org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
> at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
> at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
> at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
> at
> org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
> at
> org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
> at
> org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
> at
> org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
> at
> org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
> at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
> at picocli.CommandLine.access$1300(CommandLine.java:145)
> at
> picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
> at
> picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
> at picocli.CommandLine.execute(CommandLine.java:2078)
> at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
> at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
> at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
> Caused by: org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
> ... 30 more
>
> 2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
> ManagedStatistics is not closed properly
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
