Module Name: src Committed By: ad Date: Tue Mar 17 18:31:39 UTC 2020
Modified Files: src/sys/miscfs/genfs: genfs_io.c src/sys/rump/librump/rumpkern: vm.c src/sys/ufs/lfs: lfs_pages.c src/sys/uvm: uvm_aobj.c uvm_bio.c uvm_fault.c uvm_loan.c uvm_page.c uvm_page.h src/tests/rump/kernspace: busypage.c Log Message: Tweak the March 14th change to make page waits interlocked by pg->interlock. Remove unneeded changes and only deal with the PQ_WANTED flag, to exclude possible bugs. To generate a diff of this commit: cvs rdiff -u -r1.93 -r1.94 src/sys/miscfs/genfs/genfs_io.c cvs rdiff -u -r1.186 -r1.187 src/sys/rump/librump/rumpkern/vm.c cvs rdiff -u -r1.24 -r1.25 src/sys/ufs/lfs/lfs_pages.c cvs rdiff -u -r1.137 -r1.138 src/sys/uvm/uvm_aobj.c cvs rdiff -u -r1.105 -r1.106 src/sys/uvm/uvm_bio.c cvs rdiff -u -r1.218 -r1.219 src/sys/uvm/uvm_fault.c cvs rdiff -u -r1.97 -r1.98 src/sys/uvm/uvm_loan.c cvs rdiff -u -r1.233 -r1.234 src/sys/uvm/uvm_page.c cvs rdiff -u -r1.101 -r1.102 src/sys/uvm/uvm_page.h cvs rdiff -u -r1.7 -r1.8 src/tests/rump/kernspace/busypage.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/miscfs/genfs/genfs_io.c diff -u src/sys/miscfs/genfs/genfs_io.c:1.93 src/sys/miscfs/genfs/genfs_io.c:1.94 --- src/sys/miscfs/genfs/genfs_io.c:1.93 Sat Mar 14 20:45:23 2020 +++ src/sys/miscfs/genfs/genfs_io.c Tue Mar 17 18:31:38 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: genfs_io.c,v 1.93 2020/03/14 20:45:23 ad Exp $ */ +/* $NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1993 @@ -31,7 +31,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.93 2020/03/14 20:45:23 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.94 2020/03/17 18:31:38 ad Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -516,9 +516,9 @@ out: } uvm_pagelock(pg); uvm_pageenqueue(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); - pg->flags &= ~PG_FAKE; + pg->flags &= ~(PG_BUSY|PG_FAKE); UVM_PAGE_OWN(pg, NULL); } else if (memwrite && !overwrite && uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) { Index: src/sys/rump/librump/rumpkern/vm.c diff -u src/sys/rump/librump/rumpkern/vm.c:1.186 src/sys/rump/librump/rumpkern/vm.c:1.187 --- src/sys/rump/librump/rumpkern/vm.c:1.186 Sat Mar 14 20:23:51 2020 +++ src/sys/rump/librump/rumpkern/vm.c Tue Mar 17 18:31:38 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $ */ +/* $NetBSD: vm.c,v 1.187 2020/03/17 18:31:38 ad Exp $ */ /* * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved. @@ -41,7 +41,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.186 2020/03/14 20:23:51 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vm.c,v 1.187 2020/03/17 18:31:38 ad Exp $"); #include <sys/param.h> #include <sys/atomic.h> @@ -689,8 +689,9 @@ uvm_page_unbusy(struct vm_page **pgs, in if (pg->flags & PG_RELEASED) { uvm_pagefree(pg); } else { + pg->flags &= ~PG_BUSY; uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); } } @@ -710,17 +711,15 @@ uvm_pagewait(struct vm_page *pg, krwlock } void -uvm_pageunbusy(struct vm_page *pg) +uvm_pagewakeup(struct vm_page *pg) { - KASSERT((pg->flags & PG_BUSY) != 0); KASSERT(mutex_owned(&pg->interlock)); if ((pg->pqflags & PQ_WANTED) != 0) { pg->pqflags &= ~PQ_WANTED; wakeup(pg); } - pg->flags &= ~PG_BUSY; } void Index: src/sys/ufs/lfs/lfs_pages.c diff -u src/sys/ufs/lfs/lfs_pages.c:1.24 src/sys/ufs/lfs/lfs_pages.c:1.25 --- src/sys/ufs/lfs/lfs_pages.c:1.24 Sat Mar 14 20:45:23 2020 +++ src/sys/ufs/lfs/lfs_pages.c Tue Mar 17 18:31:38 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: lfs_pages.c,v 1.24 2020/03/14 20:45:23 ad Exp $ */ +/* $NetBSD: lfs_pages.c,v 1.25 2020/03/17 18:31:38 ad Exp $ */ /*- * Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc. @@ -60,7 +60,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.24 2020/03/14 20:45:23 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.25 2020/03/17 18:31:38 ad Exp $"); #ifdef _KERNEL_OPT #include "opt_compat_netbsd.h" @@ -348,8 +348,9 @@ check_dirty(struct lfs *fs, struct vnode pg->flags |= PG_DELWRI; } } + pg->flags &= ~PG_BUSY; uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); UVM_PAGE_OWN(pg, NULL); } Index: src/sys/uvm/uvm_aobj.c diff -u src/sys/uvm/uvm_aobj.c:1.137 src/sys/uvm/uvm_aobj.c:1.138 --- src/sys/uvm/uvm_aobj.c:1.137 Sat Mar 14 20:23:51 2020 +++ src/sys/uvm/uvm_aobj.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_aobj.c,v 1.137 2020/03/14 20:23:51 ad Exp $ */ +/* $NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $ */ /* * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and @@ -38,7 +38,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.137 2020/03/14 20:23:51 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.138 2020/03/17 18:31:39 ad Exp $"); #ifdef _KERNEL_OPT #include "opt_uvmhist.h" @@ -1300,11 +1300,12 @@ uao_pagein_page(struct uvm_aobj *aobj, i */ uvm_pagelock(pg); uvm_pageenqueue(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); - pg->flags &= ~(PG_FAKE); + pg->flags &= ~(PG_BUSY|PG_FAKE); uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); + UVM_PAGE_OWN(pg, NULL); return false; } Index: src/sys/uvm/uvm_bio.c diff -u src/sys/uvm/uvm_bio.c:1.105 src/sys/uvm/uvm_bio.c:1.106 --- src/sys/uvm/uvm_bio.c:1.105 Sat Mar 14 20:23:51 2020 +++ src/sys/uvm/uvm_bio.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_bio.c,v 1.105 2020/03/14 20:23:51 ad Exp $ */ +/* $NetBSD: uvm_bio.c,v 1.106 2020/03/17 18:31:39 ad Exp $ */ /* * Copyright (c) 1998 Chuck Silvers. @@ -34,7 +34,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.105 2020/03/14 20:23:51 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.106 2020/03/17 18:31:39 ad Exp $"); #include "opt_uvmhist.h" #include "opt_ubc.h" @@ -283,8 +283,10 @@ ubc_fault_page(const struct uvm_faultinf uvm_pagelock(pg); uvm_pageactivate(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + pg->flags &= ~PG_BUSY; + UVM_PAGE_OWN(pg, NULL); return error; } Index: src/sys/uvm/uvm_fault.c diff -u src/sys/uvm/uvm_fault.c:1.218 src/sys/uvm/uvm_fault.c:1.219 --- src/sys/uvm/uvm_fault.c:1.218 Sat Mar 14 20:23:51 2020 +++ src/sys/uvm/uvm_fault.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_fault.c,v 1.218 2020/03/14 20:23:51 ad Exp $ */ +/* $NetBSD: uvm_fault.c,v 1.219 2020/03/17 18:31:39 ad Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -32,7 +32,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.218 2020/03/14 20:23:51 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.219 2020/03/17 18:31:39 ad Exp $"); #include "opt_uvmhist.h" @@ -479,10 +479,11 @@ released: uvm_pagelock(pg); uvm_pageactivate(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); - pg->flags &= ~PG_FAKE; + pg->flags &= ~(PG_BUSY|PG_FAKE); uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); + UVM_PAGE_OWN(pg, NULL); #else panic("%s: we_own", __func__); #endif /* defined(VMSWAP) */ @@ -1851,7 +1852,7 @@ uvm_fault_lower_neighbor( * Since this page isn't the page that's actually faulting, * ignore pmap_enter() failures; it's not critical that we * enter these right now. - * NOTE: page can't be PG_RELEASED because we've + * NOTE: page can't be waited on or PG_RELEASED because we've * held the lock the whole time we've had the handle. */ KASSERT((pg->flags & PG_PAGEOUT) == 0); @@ -1988,9 +1989,11 @@ uvm_fault_lower_io( " wasn't able to relock after fault: retry", 0,0,0,0); if ((pg->flags & PG_RELEASED) == 0) { + pg->flags &= ~PG_BUSY; uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + UVM_PAGE_OWN(pg, NULL); } else { cpu_count(CPU_COUNT_FLTPGRELE, 1); uvm_pagefree(pg); @@ -2091,8 +2094,10 @@ uvm_fault_lower_direct_loan( */ uvm_pagelock(uobjpage); - uvm_pageunbusy(uobjpage); + uvm_pagewakeup(uobjpage); uvm_pageunlock(uobjpage); + uobjpage->flags &= ~PG_BUSY; + UVM_PAGE_OWN(uobjpage, NULL); uvmfault_unlockall(ufi, amap, uobj); UVMHIST_LOG(maphist, @@ -2171,9 +2176,11 @@ uvm_fault_lower_promote( * since we still hold the object lock. */ + uobjpage->flags &= ~PG_BUSY; uvm_pagelock(uobjpage); - uvm_pageunbusy(uobjpage); + uvm_pagewakeup(uobjpage); uvm_pageunlock(uobjpage); + UVM_PAGE_OWN(uobjpage, NULL); UVMHIST_LOG(maphist, " promote uobjpage %#jx to anon/page %#jx/%#jx", @@ -2260,7 +2267,7 @@ uvm_fault_lower_enter( uvm_pagelock(pg); uvm_pageenqueue(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); /* @@ -2268,7 +2275,8 @@ uvm_fault_lower_enter( * the object lock since the last time we checked. */ KASSERT((pg->flags & PG_RELEASED) == 0); - pg->flags &= ~PG_FAKE; + pg->flags &= ~(PG_BUSY|PG_FAKE); + UVM_PAGE_OWN(pg, NULL); uvmfault_unlockall(ufi, amap, uobj); if (!uvm_reclaimable()) { @@ -2291,9 +2299,10 @@ uvm_fault_lower_enter( */ KASSERT((pg->flags & PG_RELEASED) == 0); uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); - pg->flags &= ~PG_FAKE; + pg->flags &= ~(PG_BUSY|PG_FAKE); + UVM_PAGE_OWN(pg, NULL); pmap_update(ufi->orig_map->pmap); uvmfault_unlockall(ufi, amap, uobj); Index: src/sys/uvm/uvm_loan.c diff -u src/sys/uvm/uvm_loan.c:1.97 src/sys/uvm/uvm_loan.c:1.98 --- src/sys/uvm/uvm_loan.c:1.97 Sat Mar 14 20:23:51 2020 +++ src/sys/uvm/uvm_loan.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_loan.c,v 1.97 2020/03/14 20:23:51 ad Exp $ */ +/* $NetBSD: uvm_loan.c,v 1.98 2020/03/17 18:31:39 ad Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -32,7 +32,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.97 2020/03/14 20:23:51 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.98 2020/03/17 18:31:39 ad Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -712,8 +712,10 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, } uvm_pagelock(pg); uvm_pageactivate(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + pg->flags &= ~PG_BUSY; + UVM_PAGE_OWN(pg, NULL); rw_exit(uobj->vmobjlock); return (0); } @@ -751,8 +753,10 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, anon = pg->uanon; anon->an_ref++; uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + pg->flags &= ~PG_BUSY; + UVM_PAGE_OWN(pg, NULL); rw_exit(uobj->vmobjlock); **output = anon; (*output)++; @@ -781,8 +785,10 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, anon->an_page = pg; anon->an_lock = /* TODO: share amap lock */ uvm_pageactivate(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + pg->flags &= ~PG_BUSY; + UVM_PAGE_OWN(pg, NULL); rw_exit(uobj->vmobjlock); rw_exit(&anon->an_lock); **output = anon; @@ -795,8 +801,10 @@ fail: * unlock everything and bail out. */ uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + pg->flags &= ~PG_BUSY; + UVM_PAGE_OWN(pg, NULL); uvmfault_unlockall(ufi, amap, uobj, NULL); if (anon) { anon->an_ref--; @@ -851,12 +859,13 @@ again: } /* got a zero'd page. */ - pg->flags &= ~PG_FAKE; + pg->flags &= ~(PG_BUSY|PG_FAKE); pg->flags |= PG_RDONLY; uvm_pagelock(pg); uvm_pageactivate(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + UVM_PAGE_OWN(pg, NULL); } if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */ @@ -1122,6 +1131,8 @@ uvm_loanbreak(struct vm_page *uobjpage) KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY); pmap_page_protect(uobjpage, VM_PROT_NONE); /* uobj still locked */ + uobjpage->flags &= ~PG_BUSY; + UVM_PAGE_OWN(uobjpage, NULL); /* * if the page is no longer referenced by @@ -1130,7 +1141,7 @@ uvm_loanbreak(struct vm_page *uobjpage) */ uvm_pagelock2(uobjpage, pg); - uvm_pageunbusy(uobjpage); + uvm_pagewakeup(uobjpage); if (uobjpage->uanon == NULL) uvm_pagedequeue(uobjpage); Index: src/sys/uvm/uvm_page.c diff -u src/sys/uvm/uvm_page.c:1.233 src/sys/uvm/uvm_page.c:1.234 --- src/sys/uvm/uvm_page.c:1.233 Sun Mar 15 11:17:22 2020 +++ src/sys/uvm/uvm_page.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_page.c,v 1.233 2020/03/15 11:17:22 rin Exp $ */ +/* $NetBSD: uvm_page.c,v 1.234 2020/03/17 18:31:39 ad Exp $ */ /*- * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc. @@ -95,7 +95,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.233 2020/03/15 11:17:22 rin Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.234 2020/03/17 18:31:39 ad Exp $"); #include "opt_ddb.h" #include "opt_uvm.h" @@ -1708,10 +1708,14 @@ uvm_page_unbusy(struct vm_page **pgs, in pg->flags &= ~PG_RELEASED; uvm_pagefree(pg); } else { + UVMHIST_LOG(ubchist, "unbusying pg %#jx", + (uintptr_t)pg, 0, 0, 0); KASSERT((pg->flags & PG_FAKE) == 0); + pg->flags &= ~PG_BUSY; uvm_pagelock(pg); - uvm_pageunbusy(pg); + uvm_pagewakeup(pg); uvm_pageunlock(pg); + UVM_PAGE_OWN(pg, NULL); } } } @@ -1733,36 +1737,30 @@ uvm_pagewait(struct vm_page *pg, krwlock KASSERT(uvm_page_owner_locked_p(pg, false)); mutex_enter(&pg->interlock); - pg->pqflags |= PQ_WANTED; rw_exit(lock); + pg->pqflags |= PQ_WANTED; UVM_UNLOCK_AND_WAIT(pg, &pg->interlock, false, wmesg, 0); } /* - * uvm_pageunbusy: unbusy a single page + * uvm_pagewakeup: wake anyone waiting on a page * - * => page must be known PG_BUSY - * => object must be write locked * => page interlock must be held */ void -uvm_pageunbusy(struct vm_page *pg) +uvm_pagewakeup(struct vm_page *pg) { - UVMHIST_FUNC("uvm_pageunbusy"); UVMHIST_CALLED(ubchist); + UVMHIST_FUNC("uvm_pagewakeup"); UVMHIST_CALLED(ubchist); - KASSERT((pg->flags & PG_BUSY) != 0); - KASSERT(uvm_page_owner_locked_p(pg, true)); KASSERT(mutex_owned(&pg->interlock)); - UVMHIST_LOG(ubchist, "unbusying pg %#jx", (uintptr_t)pg, 0, 0, 0); + UVMHIST_LOG(ubchist, "waking pg %#jx", (uintptr_t)pg, 0, 0, 0); if ((pg->pqflags & PQ_WANTED) != 0) { wakeup(pg); pg->pqflags &= ~PQ_WANTED; } - pg->flags &= ~PG_BUSY; - UVM_PAGE_OWN(pg, NULL); } #if defined(UVM_PAGE_TRKOWN) Index: src/sys/uvm/uvm_page.h diff -u src/sys/uvm/uvm_page.h:1.101 src/sys/uvm/uvm_page.h:1.102 --- src/sys/uvm/uvm_page.h:1.101 Mon Mar 16 08:03:58 2020 +++ src/sys/uvm/uvm_page.h Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_page.h,v 1.101 2020/03/16 08:03:58 rin Exp $ */ +/* $NetBSD: uvm_page.h,v 1.102 2020/03/17 18:31:39 ad Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -375,7 +375,7 @@ void uvm_pagemarkdirty(struct vm_page *, bool uvm_pagecheckdirty(struct vm_page *, bool); bool uvm_pagereadonly_p(struct vm_page *); bool uvm_page_locked_p(struct vm_page *); -void uvm_pageunbusy(struct vm_page *); +void uvm_pagewakeup(struct vm_page *); void uvm_pagewait(struct vm_page *, krwlock_t *, const char *); int uvm_page_lookup_freelist(struct vm_page *); Index: src/tests/rump/kernspace/busypage.c diff -u src/tests/rump/kernspace/busypage.c:1.7 src/tests/rump/kernspace/busypage.c:1.8 --- src/tests/rump/kernspace/busypage.c:1.7 Sat Mar 14 20:25:46 2020 +++ src/tests/rump/kernspace/busypage.c Tue Mar 17 18:31:39 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: busypage.c,v 1.7 2020/03/14 20:25:46 ad Exp $ */ +/* $NetBSD: busypage.c,v 1.8 2020/03/17 18:31:39 ad Exp $ */ /*- * Copyright (c) 2010 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ #include <sys/cdefs.h> #if !defined(lint) -__RCSID("$NetBSD: busypage.c,v 1.7 2020/03/14 20:25:46 ad Exp $"); +__RCSID("$NetBSD: busypage.c,v 1.8 2020/03/17 18:31:39 ad Exp $"); #endif /* !lint */ #include <sys/param.h> @@ -91,9 +91,7 @@ rumptest_busypage() mutex_exit(&testpg->interlock); rw_enter(uobj->vmobjlock, RW_WRITER); - mutex_enter(&testpg->interlock); - uvm_pageunbusy(testpg); - mutex_exit(&testpg->interlock); + uvm_page_unbusy(&testpg, 1); rw_exit(uobj->vmobjlock); rv = kthread_join(newl);