Rework the lock stat counters to be scalable.
Signed-off-by: Peter Zijlstra <[EMAIL PROTECTED]>
---
include/linux/lockdep.h | 34 +++++-----
kernel/lockdep.c | 162 ++++++++++++++++++++++++++++++++++++------------
kernel/lockdep_proc.c | 136 ++++++++++------------------------------
3 files changed, 178 insertions(+), 154 deletions(-)
Index: linux-2.6/include/linux/lockdep.h
===================================================================
--- linux-2.6.orig/include/linux/lockdep.h 2007-05-23 10:23:16.000000000
+0200
+++ linux-2.6/include/linux/lockdep.h 2007-05-23 10:36:45.000000000 +0200
@@ -72,18 +72,6 @@ struct lock_class_key {
struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES];
};
-struct lock_contention_point {
- unsigned long ip;
- atomic_t count;
-};
-
-struct lock_time {
- raw_spinlock_t lock;
- unsigned long long min, max;
- unsigned long long total;
- unsigned long nr;
-};
-
/*
* The lock-class itself:
*/
@@ -128,17 +116,31 @@ struct lock_class {
const char *name;
int name_version;
+ unsigned long contention_point[4];
+};
+
#ifdef CONFIG_LOCK_STAT
- atomic_t read_contentions;
- atomic_t write_contentions;
- struct lock_contention_point contention_point[4];
+struct lock_time {
+ unsigned long long min;
+ unsigned long long max;
+ unsigned long long total;
+ unsigned long nr;
+};
+
+struct lock_class_stats {
+ unsigned long read_contentions;
+ unsigned long write_contentions;
+ unsigned long contention_point[4];
struct lock_time read_waittime;
struct lock_time write_waittime;
struct lock_time read_holdtime;
struct lock_time write_holdtime;
-#endif
};
+struct lock_class_stats lock_stats(struct lock_class *class);
+void clear_lock_stats(struct lock_class *class);
+#endif
+
/*
* Map the lock object (the lock instance) to the lock-class object.
* This is embedded into specific lock instances:
Index: linux-2.6/kernel/lockdep.c
===================================================================
--- linux-2.6.orig/kernel/lockdep.c 2007-05-23 10:23:16.000000000 +0200
+++ linux-2.6/kernel/lockdep.c 2007-05-23 11:50:13.000000000 +0200
@@ -133,6 +133,105 @@ static struct lock_list *alloc_list_entr
unsigned long nr_lock_classes;
static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
+#ifdef CONFIG_LOCK_STAT
+static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats);
+
+static int lock_contention_point(struct lock_class *class, unsigned long ip)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(class->contention_point); i++) {
+ if (class->contention_point[i] == 0) {
+ class->contention_point[i] = ip;
+ break;
+ }
+ if (class->contention_point[i] == ip)
+ break;
+ }
+
+ return i;
+}
+
+static inline void lock_contention_add(struct lock_class_stats *src,
+ struct lock_class_stats *dst)
+{
+ int i;
+
+ dst->read_contentions += src->read_contentions;
+ dst->write_contentions += src->write_contentions;
+ for (i = 0; i < ARRAY_SIZE(dst->contention_point); i++)
+ dst->contention_point[i] += src->contention_point[i];
+}
+
+static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)
+{
+ dst->min += src->min;
+ dst->max += src->max;
+ dst->total += src->total;
+ dst->nr += src->nr;
+}
+
+static void lock_time_inc(struct lock_time *lt, unsigned long long time)
+{
+ if (time > lt->max)
+ lt->max = time;
+
+ if (time < lt->min || !lt->min)
+ lt->min = time;
+
+ lt->total += time;
+ lt->nr++;
+}
+
+struct lock_class_stats lock_stats(struct lock_class *class)
+{
+ struct lock_class_stats stats;
+ int cpu, i;
+
+ memset(&stats, 0, sizeof(struct lock_class_stats));
+ for_each_possible_cpu(cpu) {
+ struct lock_class_stats *pcs =
+ &per_cpu(lock_stats, cpu)[class - lock_classes];
+
+ stats.read_contentions += pcs->read_contentions;
+ stats.write_contentions += pcs->write_contentions;
+ for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
+ stats.contention_point[i] += pcs->contention_point[i];
+
+ lock_time_add(&pcs->read_waittime, &stats.read_waittime);
+ lock_time_add(&pcs->write_waittime, &stats.write_waittime);
+
+ lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);
+ lock_time_add(&pcs->write_holdtime, &stats.write_holdtime);
+ }
+
+ return stats;
+}
+
+void clear_lock_stats(struct lock_class *class)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct lock_class_stats *cpu_stats =
+ &per_cpu(lock_stats, cpu)[class - lock_classes];
+
+ memset(cpu_stats, 0, sizeof(struct lock_class_stats));
+ }
+ memset(class->contention_point, 0, sizeof(class->contention_point));
+}
+
+static struct lock_class_stats *get_lock_stats(struct lock_class *class)
+{
+ return &get_cpu_var(lock_stats)[class - lock_classes];
+}
+
+static void put_lock_stats(struct lock_class_stats *stats)
+{
+ put_cpu_var(lock_stats);
+}
+#endif
+
/*
* We keep a global list of all lock classes. The list only grows,
* never shrinks. The list is only accessed with the lockdep
@@ -1290,12 +1389,6 @@ register_lock_class(struct lockdep_map *
INIT_LIST_HEAD(&class->lock_entry);
INIT_LIST_HEAD(&class->locks_before);
INIT_LIST_HEAD(&class->locks_after);
-#ifdef CONFIG_LOCK_STAT
- class->read_waittime.lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
- class->write_waittime.lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
- class->read_holdtime.lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
- class->write_holdtime.lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
-#endif
class->name_version = count_matching_names(class);
/*
* We use RCU's safe list-add method to make
@@ -2349,22 +2442,9 @@ static int check_unlock(struct task_stru
}
#ifdef CONFIG_LOCK_STAT
-static void lock_time_add(struct lock_time *lt, unsigned long long wt)
-{
- __raw_spin_lock(<->lock);
- if (wt > lt->max)
- lt->max = wt;
-
- if (wt < lt->min || !lt->min)
- lt->min = wt;
-
- lt->total += wt;
- lt->nr++;
- __raw_spin_unlock(<->lock);
-}
-
static void lock_release_holdtime(struct held_lock *hlock)
{
+ struct lock_class_stats *stats;
unsigned long long holdtime;
if (!lock_stat)
@@ -2372,10 +2452,14 @@ static void lock_release_holdtime(struct
holdtime = sched_clock() - hlock->holdtime_stamp;
+ stats = get_lock_stats(hlock->class);
+
if (hlock->read)
- lock_time_add(&hlock->class->read_holdtime, holdtime);
+ lock_time_inc(&stats->read_holdtime, holdtime);
else
- lock_time_add(&hlock->class->write_holdtime, holdtime);
+ lock_time_inc(&stats->write_holdtime, holdtime);
+
+ put_lock_stats(stats);
}
#else
static void lock_release_holdtime(struct held_lock *hlock)
@@ -2516,8 +2600,9 @@ __lock_contended(struct lockdep_map *loc
{
struct task_struct *curr = current;
struct held_lock *hlock, *prev_hlock;
+ struct lock_class_stats *stats;
unsigned int depth;
- int i;
+ int i, point;
depth = curr->lockdep_depth;
if (DEBUG_LOCKS_WARN_ON(!depth))
@@ -2541,22 +2626,18 @@ __lock_contended(struct lockdep_map *loc
found_it:
hlock->waittime_stamp = sched_clock();
+ point = lock_contention_point(hlock->class, ip);
+
+ stats = get_lock_stats(hlock->class);
+ if (point < ARRAY_SIZE(stats->contention_point))
+ stats->contention_point[i]++;
+
if (hlock->read)
- atomic_inc(&hlock->class->read_contentions);
+ stats->read_contentions++;
else
- atomic_inc(&hlock->class->write_contentions);
+ stats->write_contentions++;
- for (i = 0; i < ARRAY_SIZE(hlock->class->contention_point); i++) {
- if (hlock->class->contention_point[i].ip == 0) {
- hlock->class->contention_point[i].ip = ip;
- atomic_set(&hlock->class->contention_point[i].count, 1);
- break;
- }
- if (hlock->class->contention_point[i].ip == ip) {
- atomic_inc(&hlock->class->contention_point[i].count);
- break;
- }
- }
+ put_lock_stats(stats);
}
static void
@@ -2564,6 +2645,7 @@ __lock_acquired(struct lockdep_map *lock
{
struct task_struct *curr = current;
struct held_lock *hlock, *prev_hlock;
+ struct lock_class_stats *stats;
unsigned int depth;
unsigned long long now, waittime;
int i;
@@ -2596,10 +2678,14 @@ found_it:
hlock->holdtime_stamp = now;
+ stats = get_lock_stats(hlock->class);
+
if (hlock->read)
- lock_time_add(&hlock->class->read_waittime, waittime);
+ lock_time_inc(&stats->read_waittime, waittime);
else
- lock_time_add(&hlock->class->write_waittime, waittime);
+ lock_time_inc(&stats->write_waittime, waittime);
+
+ put_lock_stats(stats);
}
#endif
Index: linux-2.6/kernel/lockdep_proc.c
===================================================================
--- linux-2.6.orig/kernel/lockdep_proc.c 2007-05-23 10:23:16.000000000
+0200
+++ linux-2.6/kernel/lockdep_proc.c 2007-05-23 10:36:45.000000000 +0200
@@ -347,27 +347,28 @@ static const struct file_operations proc
#ifdef CONFIG_LOCK_STAT
static int lock_contentions_show(struct seq_file *m, void *v)
{
- struct lock_contention_point *cp;
char sym[KSYM_SYMBOL_LEN];
struct lock_class *class;
- int r, w, i;
+ struct lock_class_stats stats;
+ int i;
list_for_each_entry(class, &all_lock_classes, lock_entry) {
- r = atomic_read(&class->read_contentions);
- w = atomic_read(&class->write_contentions);
+ stats = lock_stats(class);
+
+ if (stats.read_contentions || stats.write_contentions) {
+ seq_printf(m, "%s: %lu %lu", class->name,
+ stats.write_contentions,
+ stats.read_contentions);
- if (r || w) {
- seq_printf(m, "%s: %d %d", class->name, w, r);
for (i = 0; i < ARRAY_SIZE(class->contention_point);
i++) {
- cp = &class->contention_point[i];
-
- if (cp->ip == 0)
+ if (class->contention_point[i] == 0)
break;
- sprint_symbol(sym, cp->ip);
- seq_printf(m, " [%d] [<%p>] %s",
- atomic_read(&cp->count),
- (void *)cp->ip, sym);
+ sprint_symbol(sym, class->contention_point[i]);
+ seq_printf(m, " %lu [<%p>] %s",
+ stats.contention_point[i],
+ (void *)class->contention_point[i],
+ sym);
}
seq_printf(m, "\n");
}
@@ -395,14 +396,8 @@ ssize_t lock_contentions_write(struct fi
if (c != '0')
return count;
- list_for_each_entry(class, &all_lock_classes, lock_entry) {
- atomic_set(&class->write_contentions, 0);
- atomic_set(&class->read_contentions, 0);
- for (i = 0; i < ARRAY_SIZE(class->contention_point);
- i++) {
- class->contention_point[i].ip = 0;
- }
- }
+ list_for_each_entry(class, &all_lock_classes, lock_entry)
+ clear_lock_stats(class);
}
return count;
}
@@ -417,42 +412,24 @@ static const struct file_operations proc
static void print_time(struct seq_file *m, struct lock_time *lt)
{
- unsigned long long min, total, max;
- unsigned long nr;
-
- __raw_spin_lock(<->lock);
- min = lt->min;
- total = lt->total;
- max = lt->max;
- nr = lt->nr;
- __raw_spin_unlock(<->lock);
-
- seq_printf(m, " %lu %llu %llu %llu", nr, min, max, total);
-}
-
-static void clear_time(struct lock_time *lt)
-{
- __raw_spin_lock(<->lock);
- lt->min = 0;
- lt->total = 0;
- lt->max = 0;
- lt->nr = 0;
- __raw_spin_unlock(<->lock);
+ seq_printf(m, " %lu %llu %llu %llu",
+ lt->nr, lt->min, lt->max, lt->total);
}
static int lock_waittime_show(struct seq_file *m, void *v)
{
struct lock_class *class;
- int r, w;
+ struct lock_class_stats stats;
list_for_each_entry(class, &all_lock_classes, lock_entry) {
- r = atomic_read(&class->read_contentions);
- w = atomic_read(&class->write_contentions);
+ stats = lock_stats(class);
- if (r || w) {
- seq_printf(m, "%s: %d %d", class->name, w, r);
- print_time(m, &class->write_waittime);
- print_time(m, &class->read_waittime);
+ if (stats.read_contentions || stats.write_contentions) {
+ seq_printf(m, "%s: %lu %lu", class->name,
+ stats.write_contentions,
+ stats.read_contentions);
+ print_time(m, &stats.write_waittime);
+ print_time(m, &stats.read_waittime);
seq_printf(m, "\n");
}
}
@@ -465,30 +442,9 @@ static int lock_waittime_open(struct ino
return single_open(file, lock_waittime_show, NULL);
}
-ssize_t lock_waittime_write(struct file *file, const char __user *buf,
- size_t count, loff_t *ppos)
-{
- struct lock_class *class;
- char c;
-
- if (count) {
- if (get_user(c, buf))
- return -EFAULT;
-
- if (c != '0')
- return count;
-
- list_for_each_entry(class, &all_lock_classes, lock_entry) {
- clear_time(&class->read_waittime);
- clear_time(&class->write_waittime);
- }
- }
- return count;
-}
-
static const struct file_operations proc_lock_waittime_operations = {
.open = lock_waittime_open,
- .write = lock_waittime_write,
+ .write = lock_contentions_write,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release,
@@ -497,16 +453,17 @@ static const struct file_operations proc
static int lock_holdtime_show(struct seq_file *m, void *v)
{
struct lock_class *class;
- int r, w;
+ struct lock_class_stats stats;
list_for_each_entry(class, &all_lock_classes, lock_entry) {
- r = atomic_read(&class->read_contentions);
- w = atomic_read(&class->write_contentions);
+ stats = lock_stats(class);
- if (r || w) {
- seq_printf(m, "%s: %d %d", class->name, w, r);
- print_time(m, &class->write_holdtime);
- print_time(m, &class->read_holdtime);
+ if (stats.read_contentions || stats.write_contentions) {
+ seq_printf(m, "%s: %lu %lu", class->name,
+ stats.write_contentions,
+ stats.read_contentions);
+ print_time(m, &stats.write_holdtime);
+ print_time(m, &stats.read_holdtime);
seq_printf(m, "\n");
}
}
@@ -519,30 +476,9 @@ static int lock_holdtime_open(struct ino
return single_open(file, lock_holdtime_show, NULL);
}
-ssize_t lock_holdtime_write(struct file *file, const char __user *buf,
- size_t count, loff_t *ppos)
-{
- struct lock_class *class;
- char c;
-
- if (count) {
- if (get_user(c, buf))
- return -EFAULT;
-
- if (c != '0')
- return count;
-
- list_for_each_entry(class, &all_lock_classes, lock_entry) {
- clear_time(&class->read_holdtime);
- clear_time(&class->write_holdtime);
- }
- }
- return count;
-}
-
static const struct file_operations proc_lock_holdtime_operations = {
.open = lock_holdtime_open,
- .write = lock_holdtime_write,
+ .write = lock_contentions_write,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release,
--
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
