The attached script sorts the data from the debug.mutex.prof.stats
sysctl variable according to the selected key (the default being the
mutex name)

The -g option causes the script to strip off the source file and line
and accumulate totals for each mutex.  The -r option reverses the
sorting order.  The (mutually exclusive) -a, -c, -m and -t options
sort the results by the max, total, count or average column.

Here's a list of the ten most frequently acquired mutices (over a
period of 11 days), grouped by mutex name:

des@des ~% mtxstat -gcr -l 10
         max        total        count      average name
     1077094   3634841194   3021298899            1 process lock
      200760   4475449899   2339885146            2 pool mutex
    11075293  54743743699   2127442731           26 Giant
         252   2151217659   1587924805            1 sellck
       24594   1913632993    813816142            2 vnode interlock
      442810  12872182712    716587273           18 filedesc structure
         520   2038681691    480818216            4 PCPU 16
       36114    156710765    187686674            0 bio queue
         114    270322499    185453842            1 malloc
          87    165913390    182277045            1 temp

Sorted by total time held, a few minutes later:

des@des ~% mtxstat -gtr -l 10
         max        total        count      average name
    11075293  54762076072   2128355890           26 Giant
      442810  12881479450    716922166           18 filedesc structure
      200760   4475944044   2340324507            2 pool mutex
       97625   4303676900     51655812           83 mntvnode
     1077094   3636650031   3022817522            1 process lock
         252   2152670298   1589143889            1 sellck
         520   2038861358    480872541            4 PCPU 16
       24594   1913699459    813846649            2 vnode interlock
        8418    383692889      9040940           42 xl0
          87    348538260     74251990            5 vnode_free_list

IWBNI there was a way to record how many times each mutex was
contested...

DES
-- 
Dag-Erling Smorgrav - [EMAIL PROTECTED]

#!/usr/bin/perl -Tw
#-
# Copyright (c) 2002 Dag-Erling Co´dan Sm°rgrav
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer
#    in this position and unchanged.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#      $Id$
#

use strict;
use Getopt::Std;

sub usage() {

    print(STDERR "usage: mtxstat [-gr] [-a|c|m|t] [-l limit]\n");
    exit(1);
}

MAIN:{
    my %opts;                   # Command-line options
    my $key;                    # Sort key
    my $limit;                  # Output limit
    local *PIPE;                # Pipe
    my $header;                 # Header line
    my @names;                  # Field names
    my %data;                   # Mutex data
    my @list;                   # List of entries

    getopts("acgl:mrt", \%opts)
        or usage();
    if ($opts{'a'}) {
        usage()
            if ($opts{'c'} || $opts{'m'} || $opts{'t'});
        $key = 'average';
    } elsif ($opts{'c'}) {
        usage()
            if ($opts{'m'} || $opts{'t'});
        $key = 'count';
    } elsif ($opts{'m'}) {
        usage()
            if ($opts{'t'});
        $key = 'max';
    } elsif ($opts{'t'}) {
        $key = 'total';
    }
    if ($opts{'l'}) {
        if ($opts{'l'} !~ m/^\d+$/) {
            usage();
        }
        $limit = $opts{'l'};
    }
    $ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin';
    open(PIPE, "sysctl -n debug.mutex.prof.stats|")
        or die("open(): $!\n");
    $header = <PIPE>;
    chomp($header);
    @names = split(' ', $header);
    if (defined($key) && !grep(/^$key$/, @names)) {
        die("can't find sort key '$key' in header\n");
    }
    while (<PIPE>) {
        chomp();
        my @fields = split(' ', $_, @names);
        next unless @fields;
        my %entry;
        foreach (@names) {
            $entry{$_} = ($_ eq 'name') ? shift(@fields) : 0.0 + shift(@fields);
        }
        if ($opts{'g'}) {
            $entry{'name'} =~ s/^(\S+)\s+\((.*)\)$/$2/;
        }
        my $name = $entry{'name'};
        if ($data{$name}) {
            if ($entry{'max'} > $data{$name}->{'max'}) {
                $data{$name}->{'max'} = $entry{'max'};
            }
            $data{$name}->{'total'} += $entry{'total'};
            $data{$name}->{'count'} += $entry{'count'};
            $data{$name}->{'average'} =
                $data{$name}->{'total'} / $data{$name}->{'count'};
        } else {
            $data{$name} = \%entry;
        }
    }
    if (defined($key)) {
        @list = sort({ $data{$a}->{$key} <=> $data{$b}->{$key} }
                     sort(keys(%data)));
    } else {
        @list = sort(keys(%data));
    }
    if ($opts{'r'}) {
        @list = reverse(@list);
    }
    print("$header\n");
    if ($limit) {
        while (@list > $limit) {
            pop(@list);
        }
    }
    foreach (@list) {
        printf("%12.0f %12.0f %12.0f %12.0f %s\n",
               $data{$_}->{'max'},
               $data{$_}->{'total'},
               $data{$_}->{'count'},
               $data{$_}->{'average'},
               $data{$_}->{'name'});
    }
}

Reply via email to