[PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
A bunch of what __blk_queue_bounce() was doing was problematic for the
immutable bvec work; this cleans that up and the code is quite a bit
smaller, too.
The __bio_for_each_segment() in copy_to_high_bio_irq() was changed
because that one's looping over the original bio, not the bounce bio -
since the bounce code doesn't own that bio the __ version wasn't
correct.
Signed-off-by: Kent Overstreet
CC: Jens Axboe
---
mm/bounce.c | 73 -
1 file changed, 19 insertions(+), 54 deletions(-)
diff --git a/mm/bounce.c b/mm/bounce.c
index 0420867..3068300 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -101,7 +101,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio
*from)
struct bio_vec *tovec, *fromvec;
int i;
- __bio_for_each_segment(tovec, to, i, 0) {
+ bio_for_each_segment(tovec, to, i) {
fromvec = from->bi_io_vec + i;
/*
@@ -181,78 +181,43 @@ static void bounce_end_io_read_isa(struct bio *bio, int
err)
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
mempool_t *pool)
{
- struct page *page;
- struct bio *bio = NULL;
- int i, rw = bio_data_dir(*bio_orig);
+ struct bio *bio;
+ int rw = bio_data_dir(*bio_orig);
struct bio_vec *to, *from;
+ unsigned i;
- bio_for_each_segment(from, *bio_orig, i) {
- page = from->bv_page;
+ bio_for_each_segment(from, *bio_orig, i)
+ if (page_to_pfn(from->bv_page) > queue_bounce_pfn(q))
+ goto bounce;
- /*
-* is destination page below bounce pfn?
-*/
- if (page_to_pfn(page) <= queue_bounce_pfn(q))
- continue;
-
- /*
-* irk, bounce it
-*/
- if (!bio) {
- unsigned int cnt = (*bio_orig)->bi_vcnt;
+ return;
+bounce:
+ bio = bio_clone_bioset(*bio_orig, GFP_NOIO, fs_bio_set);
- bio = bio_alloc(GFP_NOIO, cnt);
- memset(bio->bi_io_vec, 0, cnt * sizeof(struct bio_vec));
- }
-
+ bio_for_each_segment(to, bio, i) {
+ struct page *page = to->bv_page;
- to = bio->bi_io_vec + i;
+ if (page_to_pfn(page) <= queue_bounce_pfn(q))
+ continue;
- to->bv_page = mempool_alloc(pool, q->bounce_gfp);
- to->bv_len = from->bv_len;
- to->bv_offset = from->bv_offset;
inc_zone_page_state(to->bv_page, NR_BOUNCE);
+ to->bv_page = mempool_alloc(pool, q->bounce_gfp);
if (rw == WRITE) {
char *vto, *vfrom;
- flush_dcache_page(from->bv_page);
+ flush_dcache_page(page);
+
vto = page_address(to->bv_page) + to->bv_offset;
- vfrom = kmap(from->bv_page) + from->bv_offset;
+ vfrom = kmap_atomic(page) + to->bv_offset;
memcpy(vto, vfrom, to->bv_len);
- kunmap(from->bv_page);
+ kunmap_atomic(vfrom);
}
}
- /*
-* no pages bounced
-*/
- if (!bio)
- return;
-
trace_block_bio_bounce(q, *bio_orig);
- /*
-* at least one page was bounced, fill in possible non-highmem
-* pages
-*/
- __bio_for_each_segment(from, *bio_orig, i, 0) {
- to = bio_iovec_idx(bio, i);
- if (!to->bv_page) {
- to->bv_page = from->bv_page;
- to->bv_len = from->bv_len;
- to->bv_offset = from->bv_offset;
- }
- }
-
- bio->bi_bdev = (*bio_orig)->bi_bdev;
bio->bi_flags |= (1 << BIO_BOUNCED);
- bio->bi_sector = (*bio_orig)->bi_sector;
- bio->bi_rw = (*bio_orig)->bi_rw;
-
- bio->bi_vcnt = (*bio_orig)->bi_vcnt;
- bio->bi_idx = (*bio_orig)->bi_idx;
- bio->bi_size = (*bio_orig)->bi_size;
if (pool == page_pool) {
bio->bi_end_io = bounce_end_io_write;
--
1.7.12
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
A bunch of what __blk_queue_bounce() was doing was problematic for the
immutable bvec work; this cleans that up and the code is quite a bit
smaller, too.
The __bio_for_each_segment() in copy_to_high_bio_irq() was changed
because that one's looping over the original bio, not the bounce bio -
since the bounce code doesn't own that bio the __ version wasn't
correct.
Signed-off-by: Kent Overstreet koverstr...@google.com
CC: Jens Axboe ax...@kernel.dk
---
mm/bounce.c | 73 -
1 file changed, 19 insertions(+), 54 deletions(-)
diff --git a/mm/bounce.c b/mm/bounce.c
index 0420867..3068300 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -101,7 +101,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio
*from)
struct bio_vec *tovec, *fromvec;
int i;
- __bio_for_each_segment(tovec, to, i, 0) {
+ bio_for_each_segment(tovec, to, i) {
fromvec = from-bi_io_vec + i;
/*
@@ -181,78 +181,43 @@ static void bounce_end_io_read_isa(struct bio *bio, int
err)
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
mempool_t *pool)
{
- struct page *page;
- struct bio *bio = NULL;
- int i, rw = bio_data_dir(*bio_orig);
+ struct bio *bio;
+ int rw = bio_data_dir(*bio_orig);
struct bio_vec *to, *from;
+ unsigned i;
- bio_for_each_segment(from, *bio_orig, i) {
- page = from-bv_page;
+ bio_for_each_segment(from, *bio_orig, i)
+ if (page_to_pfn(from-bv_page) queue_bounce_pfn(q))
+ goto bounce;
- /*
-* is destination page below bounce pfn?
-*/
- if (page_to_pfn(page) = queue_bounce_pfn(q))
- continue;
-
- /*
-* irk, bounce it
-*/
- if (!bio) {
- unsigned int cnt = (*bio_orig)-bi_vcnt;
+ return;
+bounce:
+ bio = bio_clone_bioset(*bio_orig, GFP_NOIO, fs_bio_set);
- bio = bio_alloc(GFP_NOIO, cnt);
- memset(bio-bi_io_vec, 0, cnt * sizeof(struct bio_vec));
- }
-
+ bio_for_each_segment(to, bio, i) {
+ struct page *page = to-bv_page;
- to = bio-bi_io_vec + i;
+ if (page_to_pfn(page) = queue_bounce_pfn(q))
+ continue;
- to-bv_page = mempool_alloc(pool, q-bounce_gfp);
- to-bv_len = from-bv_len;
- to-bv_offset = from-bv_offset;
inc_zone_page_state(to-bv_page, NR_BOUNCE);
+ to-bv_page = mempool_alloc(pool, q-bounce_gfp);
if (rw == WRITE) {
char *vto, *vfrom;
- flush_dcache_page(from-bv_page);
+ flush_dcache_page(page);
+
vto = page_address(to-bv_page) + to-bv_offset;
- vfrom = kmap(from-bv_page) + from-bv_offset;
+ vfrom = kmap_atomic(page) + to-bv_offset;
memcpy(vto, vfrom, to-bv_len);
- kunmap(from-bv_page);
+ kunmap_atomic(vfrom);
}
}
- /*
-* no pages bounced
-*/
- if (!bio)
- return;
-
trace_block_bio_bounce(q, *bio_orig);
- /*
-* at least one page was bounced, fill in possible non-highmem
-* pages
-*/
- __bio_for_each_segment(from, *bio_orig, i, 0) {
- to = bio_iovec_idx(bio, i);
- if (!to-bv_page) {
- to-bv_page = from-bv_page;
- to-bv_len = from-bv_len;
- to-bv_offset = from-bv_offset;
- }
- }
-
- bio-bi_bdev = (*bio_orig)-bi_bdev;
bio-bi_flags |= (1 BIO_BOUNCED);
- bio-bi_sector = (*bio_orig)-bi_sector;
- bio-bi_rw = (*bio_orig)-bi_rw;
-
- bio-bi_vcnt = (*bio_orig)-bi_vcnt;
- bio-bi_idx = (*bio_orig)-bi_idx;
- bio-bi_size = (*bio_orig)-bi_size;
if (pool == page_pool) {
bio-bi_end_io = bounce_end_io_write;
--
1.7.12
--
To unsubscribe from this list: send the line unsubscribe linux-kernel in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Thu, Sep 20, 2012 at 05:27:06PM -0700, Tejun Heo wrote: > On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: > > A bunch of what __blk_queue_bounce() was doing was problematic for the > > immutable bvec work; this cleans that up and the code is quite a bit > > smaller, too. > > > > The __bio_for_each_segment() in copy_to_high_bio_irq() was changed > > because that one's looping over the original bio, not the bounce bio - > > since the bounce code doesn't own that bio the __ version wasn't > > correct. > > Also, I can't understand the above at all. I can think why it > wouldn't be necessary but why is it wrong because bounce code doesn't > own it? Another prep work thing - in current code, it isn't really wrong (slightly inconsistent though). But the idea is that anything that doesn't own the bio shouldn't assume anything about bi_idx; the bounce code should loop over the bio starting from wherever it was when the bio got to the bounce code, not the start of the bio. A later patch makes this clearer - __bio_for_each_segment() gets removed in favor of bio_for_each_segment_all(), and it documents that bio_for_each_segment_all() is only for code that owns the bio. -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Thu, Sep 20, 2012 at 05:25:55PM -0700, Tejun Heo wrote: > Hello, Kent. > > On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: > > A bunch of what __blk_queue_bounce() was doing was problematic for the > > immutable bvec work; this cleans that up and the code is quite a bit > > smaller, too. > > > > The __bio_for_each_segment() in copy_to_high_bio_irq() was changed > > because that one's looping over the original bio, not the bounce bio - > > since the bounce code doesn't own that bio the __ version wasn't > > correct. > > I do like the new implementation. I think the function is broken > before and after tho. Allocating from fs_bio_set from block layer is > never safe and nothing seems to prevent multiple allocators compete in > the bounce page mempool. This will need a separate bioset and the > multiple mempool allocation would have to be put inside a mutex. Yeah, I should've at least made a note of that. I should really add "audit all uses of fs_bio_set" to my todo list. > Also, how was this tested? Changed queue_bounce_pfn() to return 0, forcing all io to be bounced. -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: > A bunch of what __blk_queue_bounce() was doing was problematic for the > immutable bvec work; this cleans that up and the code is quite a bit > smaller, too. > > The __bio_for_each_segment() in copy_to_high_bio_irq() was changed > because that one's looping over the original bio, not the bounce bio - > since the bounce code doesn't own that bio the __ version wasn't > correct. Also, I can't understand the above at all. I can think why it wouldn't be necessary but why is it wrong because bounce code doesn't own it? -- tejun -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
Hello, Kent. On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: > A bunch of what __blk_queue_bounce() was doing was problematic for the > immutable bvec work; this cleans that up and the code is quite a bit > smaller, too. > > The __bio_for_each_segment() in copy_to_high_bio_irq() was changed > because that one's looping over the original bio, not the bounce bio - > since the bounce code doesn't own that bio the __ version wasn't > correct. I do like the new implementation. I think the function is broken before and after tho. Allocating from fs_bio_set from block layer is never safe and nothing seems to prevent multiple allocators compete in the bounce page mempool. This will need a separate bioset and the multiple mempool allocation would have to be put inside a mutex. Also, how was this tested? Thanks. -- tejun -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
Hello, Kent. On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: A bunch of what __blk_queue_bounce() was doing was problematic for the immutable bvec work; this cleans that up and the code is quite a bit smaller, too. The __bio_for_each_segment() in copy_to_high_bio_irq() was changed because that one's looping over the original bio, not the bounce bio - since the bounce code doesn't own that bio the __ version wasn't correct. I do like the new implementation. I think the function is broken before and after tho. Allocating from fs_bio_set from block layer is never safe and nothing seems to prevent multiple allocators compete in the bounce page mempool. This will need a separate bioset and the multiple mempool allocation would have to be put inside a mutex. Also, how was this tested? Thanks. -- tejun -- To unsubscribe from this list: send the line unsubscribe linux-kernel in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: A bunch of what __blk_queue_bounce() was doing was problematic for the immutable bvec work; this cleans that up and the code is quite a bit smaller, too. The __bio_for_each_segment() in copy_to_high_bio_irq() was changed because that one's looping over the original bio, not the bounce bio - since the bounce code doesn't own that bio the __ version wasn't correct. Also, I can't understand the above at all. I can think why it wouldn't be necessary but why is it wrong because bounce code doesn't own it? -- tejun -- To unsubscribe from this list: send the line unsubscribe linux-kernel in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Thu, Sep 20, 2012 at 05:25:55PM -0700, Tejun Heo wrote: Hello, Kent. On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: A bunch of what __blk_queue_bounce() was doing was problematic for the immutable bvec work; this cleans that up and the code is quite a bit smaller, too. The __bio_for_each_segment() in copy_to_high_bio_irq() was changed because that one's looping over the original bio, not the bounce bio - since the bounce code doesn't own that bio the __ version wasn't correct. I do like the new implementation. I think the function is broken before and after tho. Allocating from fs_bio_set from block layer is never safe and nothing seems to prevent multiple allocators compete in the bounce page mempool. This will need a separate bioset and the multiple mempool allocation would have to be put inside a mutex. Yeah, I should've at least made a note of that. I should really add audit all uses of fs_bio_set to my todo list. Also, how was this tested? Changed queue_bounce_pfn() to return 0, forcing all io to be bounced. -- To unsubscribe from this list: send the line unsubscribe linux-kernel in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
On Thu, Sep 20, 2012 at 05:27:06PM -0700, Tejun Heo wrote: On Mon, Sep 10, 2012 at 05:22:30PM -0700, Kent Overstreet wrote: A bunch of what __blk_queue_bounce() was doing was problematic for the immutable bvec work; this cleans that up and the code is quite a bit smaller, too. The __bio_for_each_segment() in copy_to_high_bio_irq() was changed because that one's looping over the original bio, not the bounce bio - since the bounce code doesn't own that bio the __ version wasn't correct. Also, I can't understand the above at all. I can think why it wouldn't be necessary but why is it wrong because bounce code doesn't own it? Another prep work thing - in current code, it isn't really wrong (slightly inconsistent though). But the idea is that anything that doesn't own the bio shouldn't assume anything about bi_idx; the bounce code should loop over the bio starting from wherever it was when the bio got to the bounce code, not the start of the bio. A later patch makes this clearer - __bio_for_each_segment() gets removed in favor of bio_for_each_segment_all(), and it documents that bio_for_each_segment_all() is only for code that owns the bio. -- To unsubscribe from this list: send the line unsubscribe linux-kernel in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
[PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
A bunch of what __blk_queue_bounce() was doing was problematic for the
immutable bvec work; this cleans that up and the code is quite a bit
smaller, too.
The __bio_for_each_segment() in copy_to_high_bio_irq() was changed
because that one's looping over the original bio, not the bounce bio -
since the bounce code doesn't own that bio the __ version wasn't
correct.
Signed-off-by: Kent Overstreet
CC: Jens Axboe
---
mm/bounce.c | 73 -
1 file changed, 19 insertions(+), 54 deletions(-)
diff --git a/mm/bounce.c b/mm/bounce.c
index 0420867..3068300 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -101,7 +101,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio
*from)
struct bio_vec *tovec, *fromvec;
int i;
- __bio_for_each_segment(tovec, to, i, 0) {
+ bio_for_each_segment(tovec, to, i) {
fromvec = from->bi_io_vec + i;
/*
@@ -181,78 +181,43 @@ static void bounce_end_io_read_isa(struct bio *bio, int
err)
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
mempool_t *pool)
{
- struct page *page;
- struct bio *bio = NULL;
- int i, rw = bio_data_dir(*bio_orig);
+ struct bio *bio;
+ int rw = bio_data_dir(*bio_orig);
struct bio_vec *to, *from;
+ unsigned i;
- bio_for_each_segment(from, *bio_orig, i) {
- page = from->bv_page;
+ bio_for_each_segment(from, *bio_orig, i)
+ if (page_to_pfn(from->bv_page) > queue_bounce_pfn(q))
+ goto bounce;
- /*
-* is destination page below bounce pfn?
-*/
- if (page_to_pfn(page) <= queue_bounce_pfn(q))
- continue;
-
- /*
-* irk, bounce it
-*/
- if (!bio) {
- unsigned int cnt = (*bio_orig)->bi_vcnt;
+ return;
+bounce:
+ bio = bio_clone_bioset(*bio_orig, GFP_NOIO, fs_bio_set);
- bio = bio_alloc(GFP_NOIO, cnt);
- memset(bio->bi_io_vec, 0, cnt * sizeof(struct bio_vec));
- }
-
+ bio_for_each_segment(to, bio, i) {
+ struct page *page = to->bv_page;
- to = bio->bi_io_vec + i;
+ if (page_to_pfn(page) <= queue_bounce_pfn(q))
+ continue;
- to->bv_page = mempool_alloc(pool, q->bounce_gfp);
- to->bv_len = from->bv_len;
- to->bv_offset = from->bv_offset;
inc_zone_page_state(to->bv_page, NR_BOUNCE);
+ to->bv_page = mempool_alloc(pool, q->bounce_gfp);
if (rw == WRITE) {
char *vto, *vfrom;
- flush_dcache_page(from->bv_page);
+ flush_dcache_page(page);
+
vto = page_address(to->bv_page) + to->bv_offset;
- vfrom = kmap(from->bv_page) + from->bv_offset;
+ vfrom = kmap_atomic(page) + to->bv_offset;
memcpy(vto, vfrom, to->bv_len);
- kunmap(from->bv_page);
+ kunmap_atomic(vfrom);
}
}
- /*
-* no pages bounced
-*/
- if (!bio)
- return;
-
trace_block_bio_bounce(q, *bio_orig);
- /*
-* at least one page was bounced, fill in possible non-highmem
-* pages
-*/
- __bio_for_each_segment(from, *bio_orig, i, 0) {
- to = bio_iovec_idx(bio, i);
- if (!to->bv_page) {
- to->bv_page = from->bv_page;
- to->bv_len = from->bv_len;
- to->bv_offset = from->bv_offset;
- }
- }
-
- bio->bi_bdev = (*bio_orig)->bi_bdev;
bio->bi_flags |= (1 << BIO_BOUNCED);
- bio->bi_sector = (*bio_orig)->bi_sector;
- bio->bi_rw = (*bio_orig)->bi_rw;
-
- bio->bi_vcnt = (*bio_orig)->bi_vcnt;
- bio->bi_idx = (*bio_orig)->bi_idx;
- bio->bi_size = (*bio_orig)->bi_size;
if (pool == page_pool) {
bio->bi_end_io = bounce_end_io_write;
--
1.7.12
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[PATCH v2 19/26] bounce: Refactor __blk_queue_bounce to not use bi_io_vec
A bunch of what __blk_queue_bounce() was doing was problematic for the
immutable bvec work; this cleans that up and the code is quite a bit
smaller, too.
The __bio_for_each_segment() in copy_to_high_bio_irq() was changed
because that one's looping over the original bio, not the bounce bio -
since the bounce code doesn't own that bio the __ version wasn't
correct.
Signed-off-by: Kent Overstreet koverstr...@google.com
CC: Jens Axboe ax...@kernel.dk
---
mm/bounce.c | 73 -
1 file changed, 19 insertions(+), 54 deletions(-)
diff --git a/mm/bounce.c b/mm/bounce.c
index 0420867..3068300 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -101,7 +101,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio
*from)
struct bio_vec *tovec, *fromvec;
int i;
- __bio_for_each_segment(tovec, to, i, 0) {
+ bio_for_each_segment(tovec, to, i) {
fromvec = from-bi_io_vec + i;
/*
@@ -181,78 +181,43 @@ static void bounce_end_io_read_isa(struct bio *bio, int
err)
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
mempool_t *pool)
{
- struct page *page;
- struct bio *bio = NULL;
- int i, rw = bio_data_dir(*bio_orig);
+ struct bio *bio;
+ int rw = bio_data_dir(*bio_orig);
struct bio_vec *to, *from;
+ unsigned i;
- bio_for_each_segment(from, *bio_orig, i) {
- page = from-bv_page;
+ bio_for_each_segment(from, *bio_orig, i)
+ if (page_to_pfn(from-bv_page) queue_bounce_pfn(q))
+ goto bounce;
- /*
-* is destination page below bounce pfn?
-*/
- if (page_to_pfn(page) = queue_bounce_pfn(q))
- continue;
-
- /*
-* irk, bounce it
-*/
- if (!bio) {
- unsigned int cnt = (*bio_orig)-bi_vcnt;
+ return;
+bounce:
+ bio = bio_clone_bioset(*bio_orig, GFP_NOIO, fs_bio_set);
- bio = bio_alloc(GFP_NOIO, cnt);
- memset(bio-bi_io_vec, 0, cnt * sizeof(struct bio_vec));
- }
-
+ bio_for_each_segment(to, bio, i) {
+ struct page *page = to-bv_page;
- to = bio-bi_io_vec + i;
+ if (page_to_pfn(page) = queue_bounce_pfn(q))
+ continue;
- to-bv_page = mempool_alloc(pool, q-bounce_gfp);
- to-bv_len = from-bv_len;
- to-bv_offset = from-bv_offset;
inc_zone_page_state(to-bv_page, NR_BOUNCE);
+ to-bv_page = mempool_alloc(pool, q-bounce_gfp);
if (rw == WRITE) {
char *vto, *vfrom;
- flush_dcache_page(from-bv_page);
+ flush_dcache_page(page);
+
vto = page_address(to-bv_page) + to-bv_offset;
- vfrom = kmap(from-bv_page) + from-bv_offset;
+ vfrom = kmap_atomic(page) + to-bv_offset;
memcpy(vto, vfrom, to-bv_len);
- kunmap(from-bv_page);
+ kunmap_atomic(vfrom);
}
}
- /*
-* no pages bounced
-*/
- if (!bio)
- return;
-
trace_block_bio_bounce(q, *bio_orig);
- /*
-* at least one page was bounced, fill in possible non-highmem
-* pages
-*/
- __bio_for_each_segment(from, *bio_orig, i, 0) {
- to = bio_iovec_idx(bio, i);
- if (!to-bv_page) {
- to-bv_page = from-bv_page;
- to-bv_len = from-bv_len;
- to-bv_offset = from-bv_offset;
- }
- }
-
- bio-bi_bdev = (*bio_orig)-bi_bdev;
bio-bi_flags |= (1 BIO_BOUNCED);
- bio-bi_sector = (*bio_orig)-bi_sector;
- bio-bi_rw = (*bio_orig)-bi_rw;
-
- bio-bi_vcnt = (*bio_orig)-bi_vcnt;
- bio-bi_idx = (*bio_orig)-bi_idx;
- bio-bi_size = (*bio_orig)-bi_size;
if (pool == page_pool) {
bio-bi_end_io = bounce_end_io_write;
--
1.7.12
--
To unsubscribe from this list: send the line unsubscribe linux-kernel in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
