[
https://issues.apache.org/jira/browse/HDDS-7197?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sammi Chen updated HDDS-7197:
-----------------------------
Description:
Currently RocksDB registers its statistics as jmx metrics with db file name,
which is same for all V3 and V2 containers. Then only the first register will
succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
ManagedStatistics is not closed properly
was:
Currently RocksDB registers its statistics as jmx metrics with db file name,
which is same for all V3 and V2 containers. Then only the first register will
succeed, others will fail with following logs.
2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
org.apache.hadoop.metrics2.MetricsException:
org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
at
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
at
org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
at
org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
at
org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
at
org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
at
org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
at
org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
at
org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
at picocli.CommandLine.access$1300(CommandLine.java:145)
at
picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
at
picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
at picocli.CommandLine.execute(CommandLine.java:2078)
at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
Caused by: org.apache.hadoop.metrics2.MetricsException:
Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
at
org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
... 30 more
> RocksDB instance statitics register as metrics collision
> --------------------------------------------------------
>
> Key: HDDS-7197
> URL: https://issues.apache.org/jira/browse/HDDS-7197
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Sammi Chen
> Assignee: Sammi Chen
> Priority: Major
>
> Currently RocksDB registers its statistics as jmx metrics with db file name,
> which is same for all V3 and V2 containers. Then only the first register will
> succeed, others will fail with following logs.
> 2022-09-01 13:25:43,867 [main] WARN util.MBeans: Error creating MBean object
> name: Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db
> org.apache.hadoop.metrics2.MetricsException:
> org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:135)
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newMBeanName(DefaultMetricsSystem.java:110)
> at org.apache.hadoop.metrics2.util.MBeans.getMBeanName(MBeans.java:165)
> at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:97)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> at
> org.apache.hadoop.hdds.HddsUtils.registerWithJmxProperties(HddsUtils.java:562)
> at org.apache.hadoop.hdds.utils.db.RDBStore.<init>(RDBStore.java:86)
> at
> org.apache.hadoop.hdds.utils.db.DBStoreBuilder.build(DBStoreBuilder.java:190)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.start(AbstractDatanodeStore.java:143)
> at
> org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore.<init>(AbstractDatanodeStore.java:99)
> at
> org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl.<init>(DatanodeStoreSchemaThreeImpl.java:54)
> at
> org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils.getUncachedDatanodeStore(BlockUtils.java:84)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.initPerDiskDBStore(HddsVolumeUtil.java:72)
> at
> org.apache.hadoop.ozone.container.common.volume.HddsVolume.loadDbStore(HddsVolume.java:255)
> at
> org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil.loadAllHddsVolumeDbStore(HddsVolumeUtil.java:92)
> at
> org.apache.hadoop.ozone.debug.container.ContainerCommands.loadContainersFromVolumes(ContainerCommands.java:147)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:47)
> at
> org.apache.hadoop.ozone.debug.container.InfoSubcommand.call(InfoSubcommand.java:32)
> at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
> at picocli.CommandLine.access$1300(CommandLine.java:145)
> at
> picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
> at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
> at
> picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
> at picocli.CommandLine.execute(CommandLine.java:2078)
> at org.apache.hadoop.hdds.cli.GenericCli.execute(GenericCli.java:100)
> at org.apache.hadoop.hdds.cli.GenericCli.run(GenericCli.java:91)
> at org.apache.hadoop.ozone.debug.OzoneDebug.main(OzoneDebug.java:57)
> Caused by: org.apache.hadoop.metrics2.MetricsException:
> Hadoop:service=Ozone,name=RocksDbStore,dbName=container.db already exists!
> at
> org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.newObjectName(DefaultMetricsSystem.java:131)
> ... 30 more
>
> 2022-09-01 13:25:44,401 [Finalizer] WARN managed.ManagedRocksObjectUtils:
> ManagedStatistics is not closed properly
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
