In the current dm-writecache, there is a trade-off in setting the watermark levels because cache blocks are evicted as soon as they have been written back: Setting it low writes back sooner and keeps more free entries to absorb a larger write burst, but setting it high retains more recently-written data to increase read hits.
This patch adds the ability to retain clean blocks in the rbtree so cache blocks can serve read hits even after writing back. This allows writing back dirty data early to make space for a large write burst AND using the entire cache capacity to service reads until that write burst actually arrives. This adds a new "clean" state to a cache block, but does not modify the eviction policy (still roughly FIFO), and read hits do not affect when a cache block is invalidated and recycled. Read hits are still opportunistic best-effort. When a cache block is reused for a new write, it needs to be removed from the rbtree and then re-inserted for the new block. The current behaviour (evict_clean_blocks=1) removes the block from the rbtree in the writeback thread as soon as writeback is done. This patch adds the option of deferring the rbtree removal until the cache block is popped off the free list, immediately before reuse. The "clean" state represents a cache block has has been put back onto the free list, but is still in the rbtree. The write_in_progress bit has been removed and rolled into a new enum of cache states: WC_INVALID: in free list, not in rbtree WC_VALID_DIRTY: not in free list, in rbtree, write_in_progress=0 WC_VALID_WRITING_BACK: not in free list, in rbtree, write_in_progress=1 WC_VALID_CLEAN: in free list, in rbtree (New state) Signed-off-by: Henry Wong <[email protected]> --- .../admin-guide/device-mapper/writecache.rst | 19 ++- drivers/md/Kconfig | 5 +- drivers/md/dm-writecache.c | 148 ++++++++++++------ 3 files changed, 123 insertions(+), 49 deletions(-) diff --git a/Documentation/admin-guide/device-mapper/writecache.rst b/Documentation/admin-guide/device-mapper/writecache.rst index 60c16b7fd5ac..d32551d986b0 100644 --- a/Documentation/admin-guide/device-mapper/writecache.rst +++ b/Documentation/admin-guide/device-mapper/writecache.rst @@ -2,9 +2,11 @@ Writecache target ================= -The writecache target caches writes on persistent memory or on SSD. It -doesn't cache reads because reads are supposed to be cached in page cache -in normal RAM. +The writecache target caches writes on persistent memory or on SSD. +Although writecache is mainly intended to buffer bursts of writes and then +drain them over time to slower storage, it also has read caching effects. +Read requests that miss the in-memory page cache but hit recently-written +data in the writecache will be served as a cache hit from the writecache. When the device is constructed, the first sector should be zeroed or the first sector should contain valid superblock from previous invocation. @@ -73,6 +75,17 @@ Constructor parameters: pause_writeback n (default: 3000) pause writeback if there was some write I/O redirected to the origin volume in the last n milliseconds + evict_clean_blocks 0|1 (default: 1) + Under the default setting (1), cache blocks are evicted + from the writecache immediately after they write back to + the origin volume. Subsequent reads of these blocks will + be serviced directly from the origin volume. + + When set to 0, blocks that have written back (clean) stay + in the cache until the block needs to be reused, allowing + these blocks to continue serving read requests for longer. + This increases the read caching effect, especially for very + large caches, even when watermark levels are set low. Status: diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index df27c7d066d2..b6d6d5c37e57 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -374,8 +374,9 @@ config DM_WRITECACHE It is intended for databases or other programs that need extremely low commit latency. - The writecache target doesn't cache reads because reads are supposed - to be cached in standard RAM. + The writecache target is mainly intended to buffer bursts of writes, + but there are opportunistic read caching effects as well (hits on + recently-written data). config DM_EBS tristate "Emulated block size target (EXPERIMENTAL)" diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 68e3fb300fc1..6277656a2ef1 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -78,12 +78,22 @@ struct wc_memory_superblock { struct wc_memory_entry entries[]; }; +typedef enum { + WC_INVALID, + WC_VALID_DIRTY, + WC_VALID_WRITING_BACK, + WC_VALID_CLEAN +} wc_entry_state; + struct wc_entry { struct rb_node rb_node; - struct list_head lru; + union { + struct rb_node freelist_rb_node; + struct list_head lru; + }; u32 age; // jiffies unsigned short wc_list_contiguous; - bool write_in_progress; + wc_entry_state state : 8; #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS uint64_t original_sector; uint64_t seq_count; @@ -167,6 +177,7 @@ struct dm_writecache { bool cleaner_set:1; bool metadata_only:1; bool pause_set:1; + bool evict_clean_blocks:1; unsigned int high_wm_percent_value; unsigned int low_wm_percent_value; @@ -671,12 +682,8 @@ static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *i rb_insert_color(&ins->rb_node, &wc->tree); list_add(&ins->lru, &wc->lru); ins->age = (u32)jiffies; -} - -static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e) -{ - list_del(&e->lru); - rb_erase(&e->rb_node, &wc->tree); + BUG_ON(ins->state != WC_INVALID); + ins->state = WC_VALID_DIRTY; } static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e) @@ -688,13 +695,13 @@ static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry wc->current_free = e; while (*node) { parent = *node; - if (&e->rb_node < *node) + if (&e->freelist_rb_node < *node) node = &parent->rb_left; else node = &parent->rb_right; } - rb_link_node(&e->rb_node, parent, node); - rb_insert_color(&e->rb_node, &wc->freetree); + rb_link_node(&e->freelist_rb_node, parent, node); + rb_insert_color(&e->freelist_rb_node, &wc->freetree); } else { list_add_tail(&e->lru, &wc->freelist); } @@ -717,6 +724,31 @@ static void writecache_max_age_timer(struct timer_list *t) } } +static void writecache_clean_entry(struct dm_writecache *wc, struct wc_entry *e) +{ + // Put entry on free list to allow reuse, but leave it in the rb tree to allow read hits on clean blocks + BUG_ON(e->state != WC_VALID_DIRTY && e->state != WC_VALID_WRITING_BACK); + e->state = WC_VALID_CLEAN; + list_del(&e->lru); + writecache_add_to_freelist(wc, e); + clear_seq_count(wc, e); + writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); + if (unlikely(waitqueue_active(&wc->freelist_wait))) + wake_up(&wc->freelist_wait); +} + +static void writecache_invalidate_entry(struct dm_writecache *wc, struct wc_entry *e) +{ + // Make a clean or invalid entry invalid. + // - Clean: Remove it from the rb tree in preparation for reusing the block. + // - Invalid: Do nothing, it is already invalid. + if (e->state == WC_VALID_CLEAN) + rb_erase(&e->rb_node, &wc->tree); + else + BUG_ON(e->state != WC_INVALID); + e->state = WC_INVALID; +} + static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector) { struct wc_entry *e; @@ -729,11 +761,11 @@ static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, s e = wc->current_free; if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector)) return NULL; - next = rb_next(&e->rb_node); - rb_erase(&e->rb_node, &wc->freetree); + next = rb_next(&e->freelist_rb_node); + rb_erase(&e->freelist_rb_node, &wc->freetree); if (unlikely(!next)) next = rb_first(&wc->freetree); - wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL; + wc->current_free = next ? container_of(next, struct wc_entry, freelist_rb_node) : NULL; } else { if (unlikely(list_empty(&wc->freelist))) return NULL; @@ -743,6 +775,7 @@ static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, s list_del(&e->lru); } wc->freelist_size--; + writecache_invalidate_entry(wc, e); // Remove clean block from rb tree if it is still there. writecache_verify_watermark(wc); @@ -751,12 +784,8 @@ static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, s static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e) { - writecache_unlink(wc, e); - writecache_add_to_freelist(wc, e); - clear_seq_count(wc, e); - writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry)); - if (unlikely(waitqueue_active(&wc->freelist_wait))) - wake_up(&wc->freelist_wait); + writecache_clean_entry(wc, e); + writecache_invalidate_entry(wc, e); } static void writecache_wait_on_freelist(struct dm_writecache *wc) @@ -791,7 +820,7 @@ static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e) static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e) { - return read_seq_count(wc, e) < wc->seq_count; + return e->state == WC_VALID_CLEAN || read_seq_count(wc, e) < wc->seq_count; } static void writecache_flush(struct dm_writecache *wc) @@ -842,10 +871,13 @@ static void writecache_flush(struct dm_writecache *wc) if (rb_node) { e2 = container_of(rb_node, struct wc_entry, rb_node); - if (read_original_sector(wc, e2) == read_original_sector(wc, e) && - likely(!e2->write_in_progress)) { - writecache_free_entry(wc, e2); - need_flush_after_free = true; + if (read_original_sector(wc, e2) == read_original_sector(wc, e)) { + if (likely(e2->state == WC_VALID_DIRTY)) { + writecache_free_entry(wc, e2); + need_flush_after_free = true; + } else if (likely(e2->state == WC_VALID_CLEAN)) { + writecache_invalidate_entry(wc, e2); // invalidate doesn't update metadata, no flush needed. + } } } if (unlikely(e->lru.prev == &wc->lru)) @@ -893,7 +925,9 @@ static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_ while (read_original_sector(wc, e) < end) { struct rb_node *node = rb_next(&e->rb_node); - if (likely(!e->write_in_progress)) { + if (likely(e->state == WC_VALID_CLEAN)) { + writecache_invalidate_entry(wc, e); + } else if (likely(e->state == WC_VALID_DIRTY)) { if (!discarded_something) { if (!WC_MODE_PMEM(wc)) { writecache_wait_for_ios(wc, READ); @@ -971,7 +1005,7 @@ static int writecache_alloc_entries(struct dm_writecache *wc) for (b = 0; b < wc->n_blocks; b++) { struct wc_entry *e = &wc->entries[b]; - e->write_in_progress = false; + e->state = WC_INVALID; cond_resched(); } @@ -1065,6 +1099,7 @@ static void writecache_resume(struct dm_target *ti) #endif for (b = 0; b < wc->n_blocks; b++) { struct wc_entry *e = &wc->entries[b]; + e->state = WC_INVALID; if (!writecache_entry_is_committed(wc, e)) { if (read_seq_count(wc, e) != -1) { @@ -1410,7 +1445,7 @@ static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio, if (read_original_sector(wc, f) != read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT)) break; - if (unlikely(f->write_in_progress)) + if (unlikely(f->state != WC_VALID_DIRTY)) break; if (writecache_entry_is_committed(wc, f)) wc->overwrote_committed = true; @@ -1448,20 +1483,24 @@ static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio return WC_MAP_ERROR; } e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0); - if (e) { + if (e && (e->state != WC_VALID_CLEAN)) { if (!writecache_entry_is_committed(wc, e)) { wc->stats.write_hits_uncommitted++; search_used = true; goto bio_copy; } wc->stats.write_hits_committed++; - if (!WC_MODE_PMEM(wc) && !e->write_in_progress) { + if (!WC_MODE_PMEM(wc) && e->state != WC_VALID_WRITING_BACK) { wc->overwrote_committed = true; search_used = true; goto bio_copy; } found_entry = true; } else { + if (e && e->state == WC_VALID_CLEAN) { + // Write hits a clean block. Drop the clean block and treat it as a miss. + writecache_invalidate_entry(wc, e); + } if (unlikely(wc->cleaner) || (wc->metadata_only && !(bio->bi_opf & REQ_META))) goto direct_write; @@ -1690,11 +1729,17 @@ static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head * i = 0; do { e = wb->wc_list[i]; - BUG_ON(!e->write_in_progress); - e->write_in_progress = false; + BUG_ON(e->state != WC_VALID_WRITING_BACK); INIT_LIST_HEAD(&e->lru); - if (!writecache_has_error(wc)) - writecache_free_entry(wc, e); + + if (likely(!writecache_has_error(wc))) { + writecache_clean_entry(wc, e); + // Invalidate clean blocks now, or defer until block leaves the free list. + if (wc->evict_clean_blocks) + writecache_invalidate_entry(wc, e); + } else { + e->state = WC_VALID_DIRTY; + } BUG_ON(!wc->writeback_size); wc->writeback_size--; n_walked++; @@ -1726,12 +1771,16 @@ static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *l e = c->e; do { - BUG_ON(!e->write_in_progress); - e->write_in_progress = false; + BUG_ON(e->state != WC_VALID_WRITING_BACK); INIT_LIST_HEAD(&e->lru); - if (!writecache_has_error(wc)) - writecache_free_entry(wc, e); - + if (likely(!writecache_has_error(wc))) { + writecache_clean_entry(wc, e); + // Invalidate clean blocks now, or defer until block leaves the free list. + if (wc->evict_clean_blocks) + writecache_invalidate_entry(wc, e); + } else { + e->state = WC_VALID_DIRTY; + } BUG_ON(!wc->writeback_size); wc->writeback_size--; e++; @@ -2006,7 +2055,7 @@ static void writecache_writeback(struct work_struct *work) e = g; } else e = container_of(wc->lru.prev, struct wc_entry, lru); - BUG_ON(e->write_in_progress); + BUG_ON(e->state == WC_VALID_WRITING_BACK); if (unlikely(!writecache_entry_is_committed(wc, e))) writecache_flush(wc); @@ -2015,7 +2064,7 @@ static void writecache_writeback(struct work_struct *work) f = container_of(node, struct wc_entry, rb_node); if (unlikely(read_original_sector(wc, f) == read_original_sector(wc, e))) { - BUG_ON(!f->write_in_progress); + BUG_ON(f->state != WC_VALID_WRITING_BACK); list_move(&e->lru, &skipped); cond_resched(); continue; @@ -2024,7 +2073,8 @@ static void writecache_writeback(struct work_struct *work) wc->writeback_size++; list_move(&e->lru, &wbl.list); wbl.size++; - e->write_in_progress = true; + BUG_ON(e->state != WC_VALID_DIRTY); + e->state = WC_VALID_WRITING_BACK; e->wc_list_contiguous = 1; f = e; @@ -2042,7 +2092,7 @@ static void writecache_writeback(struct work_struct *work) if (read_original_sector(wc, g) != read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT)) break; - if (unlikely(g->write_in_progress)) + if (unlikely(g->state == WC_VALID_WRITING_BACK)) break; if (unlikely(!writecache_entry_is_committed(wc, g))) break; @@ -2059,7 +2109,8 @@ static void writecache_writeback(struct work_struct *work) wc->writeback_size++; list_move(&g->lru, &wbl.list); wbl.size++; - g->write_in_progress = true; + BUG_ON(g->state != WC_VALID_DIRTY); + g->state = WC_VALID_WRITING_BACK; g->wc_list_contiguous = BIO_MAX_VECS; f = g; e->wc_list_contiguous++; @@ -2245,6 +2296,7 @@ static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv) mutex_init(&wc->lock); wc->max_age = MAX_AGE_UNSPECIFIED; + wc->evict_clean_blocks = true; writecache_poison_lists(wc); init_waitqueue_head(&wc->freelist_wait); timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0); @@ -2480,6 +2532,12 @@ static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv) wc->pause = msecs_to_jiffies(pause_msecs); wc->pause_set = true; wc->pause_value = pause_msecs; + } else if (!strcasecmp(string, "evict_clean_blocks") && opt_params >= 1) { + unsigned int enable; + string = dm_shift_arg(&as), opt_params--; + if (sscanf(string, "%u%c", &enable, &dummy) != 1) + goto invalid_optional; + wc->evict_clean_blocks = (enable != 0); } else { invalid_optional: r = -EINVAL; @@ -2741,6 +2799,8 @@ static void writecache_status(struct dm_target *ti, status_type_t type, DMEMIT(" metadata_only"); if (wc->pause_set) DMEMIT(" pause_writeback %u", wc->pause_value); + if (!wc->evict_clean_blocks) + DMEMIT(" evict_clean_blocks %u", wc->evict_clean_blocks); break; case STATUSTYPE_IMA: *result = '\0'; -- 2.52.0
