ChangeSet 1.2231.1.11, 2005/03/28 19:19:14-08:00, [EMAIL PROTECTED]
[PATCH] mm counter operations through macros
This patch extracts all the operations on counters protected by the page
table lock (currently rss and anon_rss) into definitions in
include/linux/sched.h. All rss operations are performed through the
following macros:
get_mm_counter(mm, member) -> Obtain the value of a counter
set_mm_counter(mm, member, value) -> Set the value of a counter
update_mm_counter(mm, member, value) -> Add to a counter
inc_mm_counter(mm, member) -> Increment a counter
dec_mm_counter(mm, member) -> Decrement a counter
With this patch it becomes easier to add new counters and it is
possible to
redefine the method of counter handling. The counters are an issue for
scalability since they are used in frequently used code paths and may
cause
cache line bouncing.
F.e. One may not use counters at all and count the pages when needed,
switch
to atomic operations if the mm_struct locking changes or split the rss
into counters that can be locally incremented.
The relevant fields of the task_struct are renamed with a leading
underscore
to catch out people who are not using the acceessor macros.
Signed-off-by: Christoph Lameter <[EMAIL PROTECTED]>
Signed-off-by: Andrew Morton <[EMAIL PROTECTED]>
Signed-off-by: Linus Torvalds <[EMAIL PROTECTED]>
arch/i386/mm/hugetlbpage.c | 6 +++---
arch/ia64/mm/hugetlbpage.c | 6 +++---
arch/m68k/atari/stram.c | 2 +-
arch/mips/kernel/irixelf.c | 2 +-
arch/ppc64/mm/hugetlbpage.c | 6 +++---
arch/sh/mm/hugetlbpage.c | 6 +++---
arch/sh64/mm/hugetlbpage.c | 6 +++---
arch/sparc64/kernel/binfmt_aout32.c | 2 +-
arch/sparc64/mm/hugetlbpage.c | 6 +++---
arch/x86_64/ia32/ia32_aout.c | 2 +-
fs/binfmt_aout.c | 2 +-
fs/binfmt_elf.c | 2 +-
fs/binfmt_elf_fdpic.c | 2 +-
fs/binfmt_flat.c | 2 +-
fs/binfmt_som.c | 2 +-
fs/exec.c | 2 +-
fs/proc/array.c | 2 +-
fs/proc/task_mmu.c | 8 +++++---
include/asm-arm/tlb.h | 4 ++--
include/asm-arm26/tlb.h | 4 ++--
include/asm-generic/tlb.h | 4 ++--
include/asm-ia64/tlb.h | 4 ++--
include/asm-sparc64/tlb.h | 4 ++--
include/linux/sched.h | 14 ++++++++++++--
kernel/acct.c | 2 +-
kernel/fork.c | 6 +++---
mm/fremap.c | 4 ++--
mm/memory.c | 22 ++++++++++++----------
mm/mmap.c | 2 +-
mm/nommu.c | 6 ++++--
mm/rmap.c | 14 +++++++-------
mm/swapfile.c | 2 +-
32 files changed, 87 insertions(+), 71 deletions(-)
diff -Nru a/arch/i386/mm/hugetlbpage.c b/arch/i386/mm/hugetlbpage.c
--- a/arch/i386/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/i386/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -46,7 +46,7 @@
{
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access) {
entry =
pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
@@ -86,7 +86,7 @@
ptepage = pte_page(entry);
get_page(ptepage);
set_pte(dst_pte, entry);
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
addr += HPAGE_SIZE;
}
return 0;
@@ -222,7 +222,7 @@
page = pte_page(pte);
put_page(page);
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm ,rss, -((end - start) >> PAGE_SHIFT));
flush_tlb_range(vma, start, end);
}
diff -Nru a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
--- a/arch/ia64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/ia64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -73,7 +73,7 @@
{
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access) {
entry =
pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
@@ -116,7 +116,7 @@
ptepage = pte_page(entry);
get_page(ptepage);
set_pte(dst_pte, entry);
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
addr += HPAGE_SIZE;
}
return 0;
@@ -246,7 +246,7 @@
put_page(page);
pte_clear(mm, address, pte);
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm, rss, - ((end - start) >> PAGE_SHIFT));
flush_tlb_range(vma, start, end);
}
diff -Nru a/arch/m68k/atari/stram.c b/arch/m68k/atari/stram.c
--- a/arch/m68k/atari/stram.c 2005-03-28 21:08:44 -08:00
+++ b/arch/m68k/atari/stram.c 2005-03-28 21:08:44 -08:00
@@ -635,7 +635,7 @@
set_pte(dir, pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
swap_free(entry);
get_page(page);
- ++vma->vm_mm->rss;
+ inc_mm_counter(vma->vm_mm, rss);
}
static inline void unswap_pmd(struct vm_area_struct * vma, pmd_t *dir,
diff -Nru a/arch/mips/kernel/irixelf.c b/arch/mips/kernel/irixelf.c
--- a/arch/mips/kernel/irixelf.c 2005-03-28 21:08:45 -08:00
+++ b/arch/mips/kernel/irixelf.c 2005-03-28 21:08:45 -08:00
@@ -692,7 +692,7 @@
/* Do this so that we can load the interpreter, if need be. We will
* change some of these later.
*/
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT);
current->mm->start_stack = bprm->p;
diff -Nru a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
--- a/arch/ppc64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/ppc64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -154,7 +154,7 @@
{
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access) {
entry =
pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
@@ -316,7 +316,7 @@
ptepage = pte_page(entry);
get_page(ptepage);
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
set_pte_at(dst, addr, dst_pte, entry);
addr += HPAGE_SIZE;
@@ -426,7 +426,7 @@
put_page(page);
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));
flush_tlb_pending();
}
diff -Nru a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c
--- a/arch/sh/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/sh/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -62,7 +62,7 @@
unsigned long i;
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access)
entry = pte_mkwrite(pte_mkdirty(mk_pte(page,
@@ -115,7 +115,7 @@
pte_val(entry) += PAGE_SIZE;
dst_pte++;
}
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
addr += HPAGE_SIZE;
}
return 0;
@@ -206,7 +206,7 @@
pte++;
}
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));
flush_tlb_range(vma, start, end);
}
diff -Nru a/arch/sh64/mm/hugetlbpage.c b/arch/sh64/mm/hugetlbpage.c
--- a/arch/sh64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/sh64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -62,7 +62,7 @@
unsigned long i;
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access)
entry = pte_mkwrite(pte_mkdirty(mk_pte(page,
@@ -115,7 +115,7 @@
pte_val(entry) += PAGE_SIZE;
dst_pte++;
}
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
addr += HPAGE_SIZE;
}
return 0;
@@ -206,7 +206,7 @@
pte++;
}
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));
flush_tlb_range(vma, start, end);
}
diff -Nru a/arch/sparc64/kernel/binfmt_aout32.c
b/arch/sparc64/kernel/binfmt_aout32.c
--- a/arch/sparc64/kernel/binfmt_aout32.c 2005-03-28 21:08:44 -08:00
+++ b/arch/sparc64/kernel/binfmt_aout32.c 2005-03-28 21:08:44 -08:00
@@ -241,7 +241,7 @@
current->mm->brk = ex.a_bss +
(current->mm->start_brk = N_BSSADDR(ex));
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
current->mm->mmap = NULL;
compute_creds(bprm);
current->flags &= ~PF_FORKNOEXEC;
diff -Nru a/arch/sparc64/mm/hugetlbpage.c b/arch/sparc64/mm/hugetlbpage.c
--- a/arch/sparc64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
+++ b/arch/sparc64/mm/hugetlbpage.c 2005-03-28 21:08:44 -08:00
@@ -68,7 +68,7 @@
unsigned long i;
pte_t entry;
- mm->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access)
entry = pte_mkwrite(pte_mkdirty(mk_pte(page,
@@ -123,7 +123,7 @@
dst_pte++;
addr += PAGE_SIZE;
}
- dst->rss += (HPAGE_SIZE / PAGE_SIZE);
+ add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
}
return 0;
@@ -213,7 +213,7 @@
pte++;
}
}
- mm->rss -= (end - start) >> PAGE_SHIFT;
+ add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT));
flush_tlb_range(vma, start, end);
}
diff -Nru a/arch/x86_64/ia32/ia32_aout.c b/arch/x86_64/ia32/ia32_aout.c
--- a/arch/x86_64/ia32/ia32_aout.c 2005-03-28 21:08:44 -08:00
+++ b/arch/x86_64/ia32/ia32_aout.c 2005-03-28 21:08:44 -08:00
@@ -313,7 +313,7 @@
(current->mm->start_brk = N_BSSADDR(ex));
current->mm->free_area_cache = TASK_UNMAPPED_BASE;
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
current->mm->mmap = NULL;
compute_creds(bprm);
current->flags &= ~PF_FORKNOEXEC;
diff -Nru a/fs/binfmt_aout.c b/fs/binfmt_aout.c
--- a/fs/binfmt_aout.c 2005-03-28 21:08:45 -08:00
+++ b/fs/binfmt_aout.c 2005-03-28 21:08:45 -08:00
@@ -317,7 +317,7 @@
(current->mm->start_brk = N_BSSADDR(ex));
current->mm->free_area_cache = current->mm->mmap_base;
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
current->mm->mmap = NULL;
compute_creds(bprm);
current->flags &= ~PF_FORKNOEXEC;
diff -Nru a/fs/binfmt_elf.c b/fs/binfmt_elf.c
--- a/fs/binfmt_elf.c 2005-03-28 21:08:44 -08:00
+++ b/fs/binfmt_elf.c 2005-03-28 21:08:44 -08:00
@@ -773,7 +773,7 @@
/* Do this so that we can load the interpreter, if need be. We will
change some of these later */
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
current->mm->free_area_cache = current->mm->mmap_base;
retval = setup_arg_pages(bprm, randomize_stack_top(STACK_TOP),
executable_stack);
diff -Nru a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
--- a/fs/binfmt_elf_fdpic.c 2005-03-28 21:08:44 -08:00
+++ b/fs/binfmt_elf_fdpic.c 2005-03-28 21:08:44 -08:00
@@ -299,7 +299,7 @@
/* do this so that we can load the interpreter, if need be
* - we will change some of these later
*/
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
#ifdef CONFIG_MMU
retval = setup_arg_pages(bprm, current->mm->start_stack,
executable_stack);
diff -Nru a/fs/binfmt_flat.c b/fs/binfmt_flat.c
--- a/fs/binfmt_flat.c 2005-03-28 21:08:44 -08:00
+++ b/fs/binfmt_flat.c 2005-03-28 21:08:44 -08:00
@@ -650,7 +650,7 @@
current->mm->start_brk = datapos + data_len + bss_len;
current->mm->brk = (current->mm->start_brk + 3) & ~3;
current->mm->context.end_brk = memp + ksize((void *) memp) -
stack_len;
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
}
if (flags & FLAT_FLAG_KTRACE)
diff -Nru a/fs/binfmt_som.c b/fs/binfmt_som.c
--- a/fs/binfmt_som.c 2005-03-28 21:08:44 -08:00
+++ b/fs/binfmt_som.c 2005-03-28 21:08:44 -08:00
@@ -259,7 +259,7 @@
create_som_tables(bprm);
current->mm->start_stack = bprm->p;
- current->mm->rss = 0;
+ set_mm_counter(current->mm, rss, 0);
#if 0
printk("(start_brk) %08lx\n" , (unsigned long) current->mm->start_brk);
diff -Nru a/fs/exec.c b/fs/exec.c
--- a/fs/exec.c 2005-03-28 21:08:44 -08:00
+++ b/fs/exec.c 2005-03-28 21:08:44 -08:00
@@ -326,7 +326,7 @@
pte_unmap(pte);
goto out;
}
- mm->rss++;
+ inc_mm_counter(mm, rss);
lru_cache_add_active(page);
set_pte_at(mm, address, pte, pte_mkdirty(pte_mkwrite(mk_pte(
page, vma->vm_page_prot))));
diff -Nru a/fs/proc/array.c b/fs/proc/array.c
--- a/fs/proc/array.c 2005-03-28 21:08:44 -08:00
+++ b/fs/proc/array.c 2005-03-28 21:08:44 -08:00
@@ -432,7 +432,7 @@
jiffies_to_clock_t(it_real_value),
start_time,
vsize,
- mm ? mm->rss : 0, /* you might want to shift this left 3 */
+ mm ? get_mm_counter(mm, rss) : 0, /* you might want to shift
this left 3 */
rsslim,
mm ? mm->start_code : 0,
mm ? mm->end_code : 0,
diff -Nru a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
--- a/fs/proc/task_mmu.c 2005-03-28 21:08:44 -08:00
+++ b/fs/proc/task_mmu.c 2005-03-28 21:08:44 -08:00
@@ -24,7 +24,7 @@
"VmPTE:\t%8lu kB\n",
(mm->total_vm - mm->reserved_vm) << (PAGE_SHIFT-10),
mm->locked_vm << (PAGE_SHIFT-10),
- mm->rss << (PAGE_SHIFT-10),
+ get_mm_counter(mm, rss) << (PAGE_SHIFT-10),
data << (PAGE_SHIFT-10),
mm->stack_vm << (PAGE_SHIFT-10), text, lib,
(PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10);
@@ -39,11 +39,13 @@
int task_statm(struct mm_struct *mm, int *shared, int *text,
int *data, int *resident)
{
- *shared = mm->rss - mm->anon_rss;
+ int rss = get_mm_counter(mm, rss);
+
+ *shared = rss - get_mm_counter(mm, anon_rss);
*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
>> PAGE_SHIFT;
*data = mm->total_vm - mm->shared_vm;
- *resident = mm->rss;
+ *resident = rss;
return mm->total_vm;
}
diff -Nru a/include/asm-arm/tlb.h b/include/asm-arm/tlb.h
--- a/include/asm-arm/tlb.h 2005-03-28 21:08:45 -08:00
+++ b/include/asm-arm/tlb.h 2005-03-28 21:08:45 -08:00
@@ -54,11 +54,11 @@
{
struct mm_struct *mm = tlb->mm;
unsigned long freed = tlb->freed;
- int rss = mm->rss;
+ int rss = get_mm_counter(mm, rss);
if (rss < freed)
freed = rss;
- mm->rss = rss - freed;
+ add_mm_counter(mm, rss, -freed);
if (freed) {
flush_tlb_mm(mm);
diff -Nru a/include/asm-arm26/tlb.h b/include/asm-arm26/tlb.h
--- a/include/asm-arm26/tlb.h 2005-03-28 21:08:45 -08:00
+++ b/include/asm-arm26/tlb.h 2005-03-28 21:08:45 -08:00
@@ -37,11 +37,11 @@
{
struct mm_struct *mm = tlb->mm;
unsigned long freed = tlb->freed;
- int rss = mm->rss;
+ int rss = get_mm_counter(mm, rss);
if (rss < freed)
freed = rss;
- mm->rss = rss - freed;
+ add_mm_counter(mm, rss, -freed);
if (freed) {
flush_tlb_mm(mm);
diff -Nru a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
--- a/include/asm-generic/tlb.h 2005-03-28 21:08:44 -08:00
+++ b/include/asm-generic/tlb.h 2005-03-28 21:08:44 -08:00
@@ -88,11 +88,11 @@
{
int freed = tlb->freed;
struct mm_struct *mm = tlb->mm;
- int rss = mm->rss;
+ int rss = get_mm_counter(mm, rss);
if (rss < freed)
freed = rss;
- mm->rss = rss - freed;
+ add_mm_counter(mm, rss, -freed);
tlb_flush_mmu(tlb, start, end);
/* keep the page table cache within bounds */
diff -Nru a/include/asm-ia64/tlb.h b/include/asm-ia64/tlb.h
--- a/include/asm-ia64/tlb.h 2005-03-28 21:08:44 -08:00
+++ b/include/asm-ia64/tlb.h 2005-03-28 21:08:44 -08:00
@@ -161,11 +161,11 @@
{
unsigned long freed = tlb->freed;
struct mm_struct *mm = tlb->mm;
- unsigned long rss = mm->rss;
+ unsigned long rss = get_mm_counter(mm, rss);
if (rss < freed)
freed = rss;
- mm->rss = rss - freed;
+ add_mm_counter(mm, rss, -freed);
/*
* Note: tlb->nr may be 0 at this point, so we can't rely on
tlb->start_addr and
* tlb->end_addr.
diff -Nru a/include/asm-sparc64/tlb.h b/include/asm-sparc64/tlb.h
--- a/include/asm-sparc64/tlb.h 2005-03-28 21:08:44 -08:00
+++ b/include/asm-sparc64/tlb.h 2005-03-28 21:08:44 -08:00
@@ -80,11 +80,11 @@
{
unsigned long freed = mp->freed;
struct mm_struct *mm = mp->mm;
- unsigned long rss = mm->rss;
+ unsigned long rss = get_mm_counter(mm, rss);
if (rss < freed)
freed = rss;
- mm->rss = rss - freed;
+ add_mm_counter(mm, rss, -freed);
tlb_flush_mmu(mp);
diff -Nru a/include/linux/sched.h b/include/linux/sched.h
--- a/include/linux/sched.h 2005-03-28 21:08:44 -08:00
+++ b/include/linux/sched.h 2005-03-28 21:08:44 -08:00
@@ -204,6 +204,12 @@
extern void arch_unmap_area(struct vm_area_struct *area);
extern void arch_unmap_area_topdown(struct vm_area_struct *area);
+#define set_mm_counter(mm, member, value) (mm)->_##member = (value)
+#define get_mm_counter(mm, member) ((mm)->_##member)
+#define add_mm_counter(mm, member, value) (mm)->_##member += (value)
+#define inc_mm_counter(mm, member) (mm)->_##member++
+#define dec_mm_counter(mm, member) (mm)->_##member--
+typedef unsigned long mm_counter_t;
struct mm_struct {
struct vm_area_struct * mmap; /* list of VMAs */
@@ -220,7 +226,7 @@
atomic_t mm_count; /* How many references to
"struct mm_struct" (users count as 1) */
int map_count; /* number of VMAs */
struct rw_semaphore mmap_sem;
- spinlock_t page_table_lock; /* Protects page tables,
mm->rss, mm->anon_rss */
+ spinlock_t page_table_lock; /* Protects page tables and
some counters */
struct list_head mmlist; /* List of maybe swapped mm's.
These are globally strung
* together off init_mm.mmlist,
and are protected
@@ -230,8 +236,12 @@
unsigned long start_code, end_code, start_data, end_data;
unsigned long start_brk, brk, start_stack;
unsigned long arg_start, arg_end, env_start, env_end;
- unsigned long rss, anon_rss, total_vm, locked_vm, shared_vm;
+ unsigned long total_vm, locked_vm, shared_vm;
unsigned long exec_vm, stack_vm, reserved_vm, def_flags, nr_ptes;
+
+ /* Special counters protected by the page_table_lock */
+ mm_counter_t _rss;
+ mm_counter_t _anon_rss;
unsigned long saved_auxv[42]; /* for /proc/PID/auxv */
diff -Nru a/kernel/acct.c b/kernel/acct.c
--- a/kernel/acct.c 2005-03-28 21:08:44 -08:00
+++ b/kernel/acct.c 2005-03-28 21:08:44 -08:00
@@ -542,7 +542,7 @@
if (delta == 0)
return;
tsk->acct_stimexpd = tsk->stime;
- tsk->acct_rss_mem1 += delta * tsk->mm->rss;
+ tsk->acct_rss_mem1 += delta * get_mm_counter(tsk->mm, rss);
tsk->acct_vm_mem1 += delta * tsk->mm->total_vm;
}
}
diff -Nru a/kernel/fork.c b/kernel/fork.c
--- a/kernel/fork.c 2005-03-28 21:08:44 -08:00
+++ b/kernel/fork.c 2005-03-28 21:08:44 -08:00
@@ -195,8 +195,8 @@
mm->mmap_cache = NULL;
mm->free_area_cache = oldmm->mmap_base;
mm->map_count = 0;
- mm->rss = 0;
- mm->anon_rss = 0;
+ set_mm_counter(mm, rss, 0);
+ set_mm_counter(mm, anon_rss, 0);
cpus_clear(mm->cpu_vm_mask);
mm->mm_rb = RB_ROOT;
rb_link = &mm->mm_rb.rb_node;
@@ -492,7 +492,7 @@
if (retval)
goto free_pt;
- mm->hiwater_rss = mm->rss;
+ mm->hiwater_rss = get_mm_counter(mm,rss);
mm->hiwater_vm = mm->total_vm;
good_mm:
diff -Nru a/mm/fremap.c b/mm/fremap.c
--- a/mm/fremap.c 2005-03-28 21:08:44 -08:00
+++ b/mm/fremap.c 2005-03-28 21:08:44 -08:00
@@ -39,7 +39,7 @@
set_page_dirty(page);
page_remove_rmap(page);
page_cache_release(page);
- mm->rss--;
+ dec_mm_counter(mm, rss);
}
}
} else {
@@ -92,7 +92,7 @@
zap_pte(mm, vma, addr, pte);
- mm->rss++;
+ inc_mm_counter(mm,rss);
flush_icache_page(vma, page);
set_pte_at(mm, addr, pte, mk_pte(page, prot));
page_add_file_rmap(page);
diff -Nru a/mm/memory.c b/mm/memory.c
--- a/mm/memory.c 2005-03-28 21:08:44 -08:00
+++ b/mm/memory.c 2005-03-28 21:08:44 -08:00
@@ -309,9 +309,9 @@
pte = pte_mkclean(pte);
pte = pte_mkold(pte);
get_page(page);
- dst_mm->rss++;
+ inc_mm_counter(dst_mm, rss);
if (PageAnon(page))
- dst_mm->anon_rss++;
+ inc_mm_counter(dst_mm, anon_rss);
set_pte_at(dst_mm, addr, dst_pte, pte);
page_dup_rmap(page);
}
@@ -475,7 +475,7 @@
if (pte_dirty(ptent))
set_page_dirty(page);
if (PageAnon(page))
- tlb->mm->anon_rss--;
+ dec_mm_counter(tlb->mm, anon_rss);
else if (pte_young(ptent))
mark_page_accessed(page);
tlb->freed++;
@@ -1219,9 +1219,9 @@
page_table = pte_offset_map(pmd, address);
if (likely(pte_same(*page_table, pte))) {
if (PageAnon(old_page))
- mm->anon_rss--;
+ dec_mm_counter(mm, anon_rss);
if (PageReserved(old_page))
- ++mm->rss;
+ inc_mm_counter(mm, rss);
else
page_remove_rmap(old_page);
flush_cache_page(vma, address, pfn);
@@ -1627,7 +1627,7 @@
if (vm_swap_full())
remove_exclusive_swap_page(page);
- mm->rss++;
+ inc_mm_counter(mm, rss);
pte = mk_pte(page, vma->vm_page_prot);
if (write_access && can_share_swap_page(page)) {
pte = maybe_mkwrite(pte_mkdirty(pte), vma);
@@ -1691,7 +1691,7 @@
spin_unlock(&mm->page_table_lock);
goto out;
}
- mm->rss++;
+ inc_mm_counter(mm, rss);
entry = maybe_mkwrite(pte_mkdirty(mk_pte(page,
vma->vm_page_prot)),
vma);
@@ -1807,7 +1807,7 @@
/* Only go through if we didn't race with anybody else... */
if (pte_none(*page_table)) {
if (!PageReserved(new_page))
- ++mm->rss;
+ inc_mm_counter(mm, rss);
flush_icache_page(vma, new_page);
entry = mk_pte(new_page, vma->vm_page_prot);
@@ -2112,8 +2112,10 @@
void update_mem_hiwater(struct task_struct *tsk)
{
if (tsk->mm) {
- if (tsk->mm->hiwater_rss < tsk->mm->rss)
- tsk->mm->hiwater_rss = tsk->mm->rss;
+ unsigned long rss = get_mm_counter(tsk->mm, rss);
+
+ if (tsk->mm->hiwater_rss < rss)
+ tsk->mm->hiwater_rss = rss;
if (tsk->mm->hiwater_vm < tsk->mm->total_vm)
tsk->mm->hiwater_vm = tsk->mm->total_vm;
}
diff -Nru a/mm/mmap.c b/mm/mmap.c
--- a/mm/mmap.c 2005-03-28 21:08:44 -08:00
+++ b/mm/mmap.c 2005-03-28 21:08:44 -08:00
@@ -1978,7 +1978,7 @@
vma = mm->mmap;
mm->mmap = mm->mmap_cache = NULL;
mm->mm_rb = RB_ROOT;
- mm->rss = 0;
+ set_mm_counter(mm, rss, 0);
mm->total_vm = 0;
mm->locked_vm = 0;
diff -Nru a/mm/nommu.c b/mm/nommu.c
--- a/mm/nommu.c 2005-03-28 21:08:44 -08:00
+++ b/mm/nommu.c 2005-03-28 21:08:44 -08:00
@@ -961,9 +961,11 @@
void update_mem_hiwater(struct task_struct *tsk)
{
+ unsigned long rss = get_mm_counter(tsk->mm, rss);
+
if (likely(tsk->mm)) {
- if (tsk->mm->hiwater_rss < tsk->mm->rss)
- tsk->mm->hiwater_rss = tsk->mm->rss;
+ if (tsk->mm->hiwater_rss < rss)
+ tsk->mm->hiwater_rss = rss;
if (tsk->mm->hiwater_vm < tsk->mm->total_vm)
tsk->mm->hiwater_vm = tsk->mm->total_vm;
}
diff -Nru a/mm/rmap.c b/mm/rmap.c
--- a/mm/rmap.c 2005-03-28 21:08:45 -08:00
+++ b/mm/rmap.c 2005-03-28 21:08:45 -08:00
@@ -257,7 +257,7 @@
pte_t *pte;
int referenced = 0;
- if (!mm->rss)
+ if (!get_mm_counter(mm, rss))
goto out;
address = vma_address(page, vma);
if (address == -EFAULT)
@@ -436,7 +436,7 @@
BUG_ON(PageReserved(page));
BUG_ON(!anon_vma);
- vma->vm_mm->anon_rss++;
+ inc_mm_counter(vma->vm_mm, anon_rss);
anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
index = (address - vma->vm_start) >> PAGE_SHIFT;
@@ -509,7 +509,7 @@
pte_t pteval;
int ret = SWAP_AGAIN;
- if (!mm->rss)
+ if (!get_mm_counter(mm, rss))
goto out;
address = vma_address(page, vma);
if (address == -EFAULT)
@@ -595,10 +595,10 @@
}
set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
BUG_ON(pte_file(*pte));
- mm->anon_rss--;
+ dec_mm_counter(mm, anon_rss);
}
- mm->rss--;
+ inc_mm_counter(mm, rss);
page_remove_rmap(page);
page_cache_release(page);
@@ -703,7 +703,7 @@
page_remove_rmap(page);
page_cache_release(page);
- mm->rss--;
+ dec_mm_counter(mm, rss);
(*mapcount)--;
}
@@ -802,7 +802,7 @@
if (vma->vm_flags & (VM_LOCKED|VM_RESERVED))
continue;
cursor = (unsigned long) vma->vm_private_data;
- while (vma->vm_mm->rss &&
+ while (get_mm_counter(vma->vm_mm, rss) &&
cursor < max_nl_cursor &&
cursor < vma->vm_end - vma->vm_start) {
try_to_unmap_cluster(cursor, &mapcount, vma);
diff -Nru a/mm/swapfile.c b/mm/swapfile.c
--- a/mm/swapfile.c 2005-03-28 21:08:44 -08:00
+++ b/mm/swapfile.c 2005-03-28 21:08:44 -08:00
@@ -422,7 +422,7 @@
static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
unsigned long addr, swp_entry_t entry, struct page *page)
{
- vma->vm_mm->rss++;
+ inc_mm_counter(vma->vm_mm, rss);
get_page(page);
set_pte_at(vma->vm_mm, addr, pte,
pte_mkold(mk_pte(page, vma->vm_page_prot)));
-
To unsubscribe from this list: send the line "unsubscribe bk-commits-head" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
