Added: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeActivtyMBean.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeActivtyMBean.java?rev=726900&view=auto ============================================================================== --- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeActivtyMBean.java (added) +++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeActivtyMBean.java Mon Dec 15 17:02:51 2008 @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.metrics; + +import javax.management.ObjectName; + +import org.apache.hadoop.metrics.util.MBeanUtil; +import org.apache.hadoop.metrics.util.MetricsDynamicMBeanBase; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +/** + * + * This is the JMX MBean for reporting the NameNode Activity. + * The MBean is register using the name + * "hadoop:service=NameNode,name=NameNodeActivity" + * + * Many of the activity metrics are sampled and averaged on an interval + * which can be specified in the metrics config file. + * <p> + * For the metrics that are sampled and averaged, one must specify + * a metrics context that does periodic update calls. Most metrics contexts do. + * The default Null metrics context however does NOT. So if you aren't + * using any other metrics context then you can turn on the viewing and averaging + * of sampled metrics by specifying the following two lines + * in the hadoop-meterics.properties file: +* <pre> + * dfs.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread + * dfs.period=10 + * </pre> + *<p> + * Note that the metrics are collected regardless of the context used. + * The context with the update thread is used to average the data periodically + * + * + * + * Impl details: We use a dynamic mbean that gets the list of the metrics + * from the metrics registry passed as an argument to the constructor + */ + +public class NameNodeActivtyMBean extends MetricsDynamicMBeanBase { + final private ObjectName mbeanName; + + protected NameNodeActivtyMBean(final MetricsRegistry mr) { + super(mr, "Activity statistics at the NameNode"); + mbeanName = MBeanUtil.registerMBean("NameNode", "NameNodeActivity", this); + } + + public void shutdown() { + if (mbeanName != null) + MBeanUtil.unregisterMBean(mbeanName); + } +}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java?rev=726900&r1=726899&r2=726900&view=diff ============================================================================== --- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java (original) +++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java Mon Dec 15 17:02:51 2008 @@ -21,16 +21,17 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeStatistics; import org.apache.hadoop.metrics.*; import org.apache.hadoop.metrics.jvm.JvmMetrics; +import org.apache.hadoop.metrics.util.MetricsBase; import org.apache.hadoop.metrics.util.MetricsIntValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt; import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; /** * - * This class is for maintaining the various NameNode statistics + * This class is for maintaining the various NameNode activity statistics * and publishing them through the metrics interfaces. * This also registers the JMX MBean for RPC. * <p> @@ -43,29 +44,41 @@ public class NameNodeMetrics implements Updater { private static Log log = LogFactory.getLog(NameNodeMetrics.class); private final MetricsRecord metricsRecord; + public MetricsRegistry registry = new MetricsRegistry(); - private NameNodeStatistics namenodeStats; + private NameNodeActivtyMBean namenodeActivityMBean; - public MetricsTimeVaryingInt numFilesCreated = new MetricsTimeVaryingInt("FilesCreated"); - public MetricsTimeVaryingInt numFilesAppended = new MetricsTimeVaryingInt("FilesAppended"); - public MetricsTimeVaryingInt numGetBlockLocations = new MetricsTimeVaryingInt("GetBlockLocations"); - public MetricsTimeVaryingInt numFilesRenamed = new MetricsTimeVaryingInt("FilesRenamed"); + public MetricsTimeVaryingInt numFilesCreated = + new MetricsTimeVaryingInt("FilesCreated", registry); + public MetricsTimeVaryingInt numFilesAppended = + new MetricsTimeVaryingInt("FilesAppended", registry); + public MetricsTimeVaryingInt numGetBlockLocations = + new MetricsTimeVaryingInt("GetBlockLocations", registry); + public MetricsTimeVaryingInt numFilesRenamed = + new MetricsTimeVaryingInt("FilesRenamed", registry); public MetricsTimeVaryingInt numGetListingOps = - new MetricsTimeVaryingInt("GetListingOps"); + new MetricsTimeVaryingInt("GetListingOps", registry); public MetricsTimeVaryingInt numCreateFileOps = - new MetricsTimeVaryingInt("CreateFileOps"); + new MetricsTimeVaryingInt("CreateFileOps", registry); public MetricsTimeVaryingInt numDeleteFileOps = - new MetricsTimeVaryingInt("DeleteFileOps"); + new MetricsTimeVaryingInt("DeleteFileOps", registry); public MetricsTimeVaryingInt numAddBlockOps = - new MetricsTimeVaryingInt("AddBlockOps"); + new MetricsTimeVaryingInt("AddBlockOps", registry); - public MetricsTimeVaryingRate transactions = new MetricsTimeVaryingRate("Transactions"); - public MetricsTimeVaryingRate syncs = new MetricsTimeVaryingRate("Syncs"); - public MetricsTimeVaryingRate blockReport = new MetricsTimeVaryingRate("blockReport"); - public MetricsIntValue safeModeTime = new MetricsIntValue("SafemodeTime"); + public MetricsTimeVaryingRate transactions = + new MetricsTimeVaryingRate("Transactions", registry, "Journal Transaction"); + public MetricsTimeVaryingRate syncs = + new MetricsTimeVaryingRate("Syncs", registry, "Journal Sync"); + public MetricsTimeVaryingInt transactionsBatchedInSync = + new MetricsTimeVaryingInt("JournalTransactionsBatchedInSync", registry, "Journal Transactions Batched In Sync"); + public MetricsTimeVaryingRate blockReport = + new MetricsTimeVaryingRate("blockReport", registry, "Block Report"); + public MetricsIntValue safeModeTime = + new MetricsIntValue("SafemodeTime", registry, "Duration in SafeMode at Startup"); public MetricsIntValue fsImageLoadTime = - new MetricsIntValue("fsImageLoadTime"); - public MetricsIntValue numBlocksCorrupted = new MetricsIntValue("BlocksCorrupted"); + new MetricsIntValue("fsImageLoadTime", registry, "Time loading FS Image at Startup"); + public MetricsIntValue numBlocksCorrupted = + new MetricsIntValue("BlocksCorrupted", registry); public NameNodeMetrics(Configuration conf, NameNode nameNode) { @@ -75,7 +88,7 @@ // Now the Mbean for the name node - this alos registers the MBean - namenodeStats = new NameNodeStatistics(this); + namenodeActivityMBean = new NameNodeActivtyMBean(registry); // Create a record for NameNode metrics MetricsContext metricsContext = MetricsUtil.getContext("dfs"); @@ -89,8 +102,8 @@ public void shutdown() { - if (namenodeStats != null) - namenodeStats.shutdown(); + if (namenodeActivityMBean != null) + namenodeActivityMBean.shutdown(); } /** @@ -99,21 +112,9 @@ */ public void doUpdates(MetricsContext unused) { synchronized (this) { - numFilesCreated.pushMetric(metricsRecord); - numFilesAppended.pushMetric(metricsRecord); - numGetBlockLocations.pushMetric(metricsRecord); - numFilesRenamed.pushMetric(metricsRecord); - numGetListingOps.pushMetric(metricsRecord); - numCreateFileOps.pushMetric(metricsRecord); - numDeleteFileOps.pushMetric(metricsRecord); - numAddBlockOps.pushMetric(metricsRecord); - - transactions.pushMetric(metricsRecord); - syncs.pushMetric(metricsRecord); - blockReport.pushMetric(metricsRecord); - safeModeTime.pushMetric(metricsRecord); - fsImageLoadTime.pushMetric(metricsRecord); - numBlocksCorrupted.pushMetric(metricsRecord); + for (MetricsBase m : registry.getMetricsList()) { + m.pushMetric(metricsRecord); + } } metricsRecord.update(); } @@ -123,5 +124,4 @@ syncs.resetMinMax(); blockReport.resetMinMax(); } - } Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java?rev=726900&r1=726899&r2=726900&view=diff ============================================================================== --- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java (original) +++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatistics.java Mon Dec 15 17:02:51 2008 @@ -1,220 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode.metrics; - -import javax.management.ObjectName; - -import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; -import org.apache.hadoop.metrics.util.MBeanUtil; - -/** - * - * This is the implementation of the Name Node JMX MBean - * - */ -public class NameNodeStatistics implements NameNodeStatisticsMBean { - private NameNodeMetrics myMetrics; - private ObjectName mbeanName; - - /** - * This constructs and registers the NameNodeStatisticsMBean - * @param nameNodeMetrics - the metrics from which the mbean gets its info - */ - public NameNodeStatistics(NameNodeMetrics nameNodeMetrics) { - myMetrics = nameNodeMetrics; - mbeanName = MBeanUtil.registerMBean("NameNode", "NameNodeStatistics", this); - } - - /** - * Shuts down the statistics - * - unregisters the mbean - */ - public void shutdown() { - if (mbeanName != null) - MBeanUtil.unregisterMBean(mbeanName); - } - - /** - * @inheritDoc - */ - public long getBlockReportAverageTime() { - return myMetrics.blockReport.getPreviousIntervalAverageTime(); - } - - /** - * @inheritDoc - */ - public long getBlockReportMaxTime() { - return myMetrics.blockReport.getMaxTime(); - } - - /** - * @inheritDoc - */ - public long getBlockReportMinTime() { - return myMetrics.blockReport.getMinTime(); - } - - /** - * @inheritDoc - */ - public int getBlockReportNum() { - return myMetrics.blockReport.getPreviousIntervalNumOps(); - } - - /** - * @inheritDoc - */ - public long getJournalTransactionAverageTime() { - return myMetrics.transactions.getPreviousIntervalAverageTime(); - } - - /** - * @inheritDoc - */ - public int getJournalTransactionNum() { - return myMetrics.transactions.getPreviousIntervalNumOps(); - } - - /** - * @inheritDoc - */ - public long getJournalTransactionMaxTime() { - return myMetrics.transactions.getMaxTime(); - } - - /** - * @inheritDoc - */ - public long getJournalTransactionMinTime() { - return myMetrics.transactions.getMinTime(); - } - - /** - * @inheritDoc - */ - public long getJournalSyncAverageTime() { - return myMetrics.syncs.getPreviousIntervalAverageTime(); - } - - /** - * @inheritDoc - */ - public long getJournalSyncMaxTime() { - return myMetrics.syncs.getMaxTime(); - } - - - /** - * @inheritDoc - */ - public long getJournalSyncMinTime() { - return myMetrics.syncs.getMinTime(); - } - - /** - * @inheritDoc - */ - public int getJournalSyncNum() { - return myMetrics.syncs.getPreviousIntervalNumOps(); - } - - - /** - * @inheritDoc - */ - public int getSafemodeTime() { - return myMetrics.safeModeTime.get(); - } - - /** - * @inheritDoc - */ - public int getFSImageLoadTime() { - return myMetrics.fsImageLoadTime.get(); - } - - /** - * @inheritDoc - */ - public void resetAllMinMax() { - myMetrics.resetAllMinMax(); - } - - /** - * @inheritDoc - */ - public int getNumFilesCreated() { - return myMetrics.numFilesCreated.getPreviousIntervalValue(); - } - - /** - *...@deprecated call getNumGetListingOps() instead - */ - @Deprecated - public int getNumFilesListed() { - return getNumGetListingOps(); - } - - /** - * @inheritDoc - */ - public int getNumGetListingOps() { - return myMetrics.numGetListingOps.getPreviousIntervalValue(); - } - - /** - * @inheritDoc - */ - public int getNumCreateFileOps() { - return myMetrics.numCreateFileOps.getPreviousIntervalValue(); - } - - /** - * @inheritDoc - */ - public int getNumDeleteFileOps() { - return myMetrics.numDeleteFileOps.getPreviousIntervalValue(); - } - - /** - * @inheritDoc - */ - public int getNumAddBlockOps() { - return myMetrics.numAddBlockOps.getPreviousIntervalValue(); - } - - /** @inheritDoc */ - public int getNumGetBlockLocations() { - return myMetrics.numGetBlockLocations.getPreviousIntervalValue(); - } - - /** - * @inheritDoc - */ - public int getNumFilesRenamed() { - return myMetrics.numFilesRenamed.getPreviousIntervalValue(); - } - - /** - * @inheritDoc - */ - public int getNumFilesAppended() { - return myMetrics.numFilesAppended.getPreviousIntervalValue(); - } -} Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java?rev=726900&r1=726899&r2=726900&view=diff ============================================================================== --- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java (original) +++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeStatisticsMBean.java Mon Dec 15 17:02:51 2008 @@ -1,187 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs.server.namenode.metrics; - -/** - * - * This is the JMX management interface for getting runtime statistics of - * the name node. - * Many of the statistics are sampled and averaged on an interval - * which can be specified in the config file. - * <p> - * For the statistics that are sampled and averaged, one must specify - * a metrics context that does periodic update calls. Most do. - * The default Null metrics context however does NOT. So if you aren't - * using any other metrics context then you can turn on the viewing and averaging - * of sampled metrics by specifying the following two lines - * in the hadoop-meterics.properties file: - * <pre> - * dfs.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread - * dfs.period=10 - * </pre> - *<p> - * Note that the metrics are collected regardless of the context used. - * The context with the update thread is used to average the data periodically. - * <p> - * Name Node Status info is report in another MBean - * @see org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean - * - */ -public interface NameNodeStatisticsMBean { - - /** - * The time spent in the Safemode at startup - * @return time in msec - */ - int getSafemodeTime(); - - /** - * Time spent loading the FS Image at startup - * @return time in msec - */ - int getFSImageLoadTime(); - - /** - * Number of Journal Transactions in the last interval - * @return number of operations - */ - int getJournalTransactionNum(); - - /** - * Average time for Journal transactions in last interval - * @return time in msec - */ - long getJournalTransactionAverageTime(); - - /** - * The Minimum Journal Transaction Time since reset was called - * @return time in msec - */ - long getJournalTransactionMinTime(); - - /** - * The Maximum Journal Transaction Time since reset was called - * @return time in msec - */ - long getJournalTransactionMaxTime(); - - /** - * Number of block Reports processed in the last interval - * @return number of operations - */ - int getBlockReportNum(); - - /** - * Average time for Block Report Processing in last interval - * @return time in msec - */ - long getBlockReportAverageTime(); - - /** - * The Minimum Block Report Processing Time since reset was called - * @return time in msec - */ - long getBlockReportMinTime(); - - /** - * The Maximum Block Report Processing Time since reset was called - * @return time in msec - */ - long getBlockReportMaxTime(); - - /** - * Number of Journal Syncs in the last interval - * @return number of operations - */ - int getJournalSyncNum(); - - /** - * Average time for Journal Sync in last interval - * @return time in msec - */ - long getJournalSyncAverageTime(); - - /** - * The Minimum Journal Sync Time since reset was called - * @return time in msec - */ - long getJournalSyncMinTime(); - - /** - * The Maximum Journal Sync Time since reset was called - * @return time in msec - */ - long getJournalSyncMaxTime(); - - /** - * Reset all min max times - */ - void resetAllMinMax(); - - /** - * Number of files created in the last interval - * @return number of operations - */ - int getNumFilesCreated(); - - /** - * Number of - * {...@link org.apache.hadoop.hdfs.server.namenode.NameNode#getBlockLocations(String,long,long)} - * @return number of operations - */ - int getNumGetBlockLocations(); - - /** - * Number of files renamed in the last interval - * @return number of operations - */ - int getNumFilesRenamed(); - - /** - * Number of files listed in the last interval - * @return number of operations - * @deprecated Use getNumGetListingOps() instead - */ - @Deprecated - int getNumFilesListed(); - - /** - * Number of files listed in the last interval - * @return number of operations - */ - int getNumGetListingOps(); - - /** - * Number of file creation operations in the last interval - * @return number of file creation operations - */ - int getNumCreateFileOps(); - - /** - * Number of file deletion operations in the last interval - * @return number of file deletion operations - */ - int getNumDeleteFileOps(); - - /** - * Number of add block operations in the last interval - * @return number of add block operations - */ - int getNumAddBlockOps(); -} Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=726900&r1=726899&r2=726900&view=diff ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original) +++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Mon Dec 15 17:02:51 2008 @@ -606,26 +606,29 @@ } private ObjectName mbeanName; + + + /** - * Register the FSDataset MBean - * @param storageId - * - * We use storage id for MBean name since a minicluster within a single + * Register the FSDataset MBean using the name + * "hadoop:service=DataNode,name=FSDatasetState-<storageid>" + * We use storage id for MBean name since a minicluster within a single * Java VM may have multiple Simulated Datanodes. */ - void registerMBean(String storageId) { - // We wrap to bypass standard mbean naming convention. - // This wrapping can be removed in java 6 as Java6 is more flexible in + void registerMBean(final String storageId) { + // We wrap to bypass standard mbean naming convetion. + // This wraping can be removed in java 6 as it is more flexible in // package naming for mbeans and their impl. StandardMBean bean; + try { bean = new StandardMBean(this,FSDatasetMBean.class); - mbeanName = MBeanUtil.registerMBean("DataNode-"+ storageId, - "FSDatasetStatus", bean); + mbeanName = MBeanUtil.registerMBean("DataNode", + "FSDatasetState-" + storageId, bean); } catch (NotCompliantMBeanException e) { e.printStackTrace(); } - + DataNode.LOG.info("Registered FSDatasetStatusMBean"); } Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java?rev=726900&r1=726899&r2=726900&view=diff ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java (original) +++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java Mon Dec 15 17:02:51 2008 @@ -24,7 +24,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; -import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeStatistics; import org.apache.hadoop.conf.Configuration; import junit.framework.TestCase; @@ -43,9 +42,7 @@ assertEquals(datanodes.size(), 1); DataNode datanode = datanodes.get(0); DataNodeMetrics metrics = datanode.getMetrics(); - DataNodeStatistics statistics = new DataNodeStatistics( - metrics, datanode.dnRegistration.storageID); - assertEquals(LONG_FILE_LEN, statistics.getBytesWritten()); + assertEquals(LONG_FILE_LEN, metrics.bytesWritten.getCurrentIntervalValue()); } finally { if (cluster != null) {cluster.shutdown();} }
