The branch main has been updated by mav: URL: https://cgit.FreeBSD.org/src/commit/?id=b55783976945e900889d7ce9d5055a069de7441f
commit b55783976945e900889d7ce9d5055a069de7441f Author: Alexander Motin <[email protected]> AuthorDate: 2026-09-01 15:39:13 +0000 Commit: Alexander Motin <[email protected]> CommitDate: 2026-09-01 15:48:55 +0000 gvirstor: Modernize the I/O path - Add unmapped I/O support. The only case when the code needs data access is BIO_READ returning zeroes for unallocated space. - Add BIO_FLUSH support. Just send it to all allocated components. - Add BIO_DELETE support. While current design does not allow freeing allocated blocks, at least pass it to underlying providers. - Add direct I/O completion support. - Add rotation rate reporting. - Fix few minor issues. --- sys/geom/virstor/g_virstor.c | 306 ++++++++++++++++++++++++++++++++++++------- sys/geom/virstor/g_virstor.h | 3 + 2 files changed, 262 insertions(+), 47 deletions(-) diff --git a/sys/geom/virstor/g_virstor.c b/sys/geom/virstor/g_virstor.c index 1490ed103329..46c5216f66c1 100644 --- a/sys/geom/virstor/g_virstor.c +++ b/sys/geom/virstor/g_virstor.c @@ -46,9 +46,13 @@ #include <sys/proc.h> #include <sys/kthread.h> #include <sys/mutex.h> +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_page.h> #include <vm/uma.h> #include <geom/geom.h> #include <geom/geom_dbg.h> +#include <geom/geom_disk.h> #include <geom/virstor/g_virstor.h> #include <geom/virstor/g_virstor_md.h> @@ -131,6 +135,9 @@ static void fill_metadata(struct g_virstor_softc *, struct g_virstor_metadata *, static void g_virstor_orphan(struct g_consumer *); static int g_virstor_access(struct g_provider *, int, int, int); +static void g_virstor_passdown(struct g_virstor_softc *, struct bio *); +static void g_virstor_candelete(struct g_virstor_softc *, struct bio *); +static void g_virstor_rotation_rate(struct g_virstor_softc *, struct bio *); static void g_virstor_start(struct bio *); static void g_virstor_dumpconf(struct sbuf *, const char *, struct g_geom *, struct g_consumer *, struct g_provider *); @@ -303,6 +310,10 @@ virstor_ctl_add(struct gctl_req *req, struct g_class *cp) gctl_error(req, "Virstor %s is incomplete", sc->geom->name); return; } + if (sc->provider == NULL) { + gctl_error(req, "Virstor %s is not running", sc->geom->name); + return; + } fcp = sc->components[0].gcons; added = 0; @@ -312,6 +323,8 @@ virstor_ctl_add(struct gctl_req *req, struct g_class *cp) char aname[8]; struct g_provider *pp; struct g_consumer *cp; + uint16_t rr; + int cd; u_int nc; u_int j; @@ -327,12 +340,33 @@ virstor_ctl_add(struct gctl_req *req, struct g_class *cp) g_topology_unlock(); return; } + if (fcp->provider->sectorsize != pp->sectorsize) { + gctl_error(req, "Sector size doesn't fit for %s", + pp->name); + g_topology_unlock(); + return; + } + if (pp->mediasize / sc->chunk_size < 4) { + gctl_error(req, "Provider too small: %s", pp->name); + g_topology_unlock(); + return; + } + for (j = 0; j < sc->n_components; j++) { + if (strcmp(sc->components[j].gcons->provider->name, + pp->name) == 0) { + gctl_error(req, "Component %s already in %s", + pp->name, sc->geom->name); + g_topology_unlock(); + return; + } + } cp = g_new_consumer(sc->geom); if (cp == NULL) { gctl_error(req, "Cannot create consumer"); g_topology_unlock(); return; } + cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; error = g_attach(cp, pp); if (error != 0) { gctl_error(req, "Cannot attach a consumer to %s", @@ -341,28 +375,22 @@ virstor_ctl_add(struct gctl_req *req, struct g_class *cp) g_topology_unlock(); return; } + /* Cascade candelete and rotation rate. */ + cd = 0; + rr = DISK_RR_UNKNOWN; + if (g_access(cp, 1, 0, 0) == 0) { + if (g_getattr("GEOM::candelete", cp, &cd) != 0) + cd = 0; + if (g_getattr("GEOM::rotation_rate", cp, &rr) != 0) + rr = DISK_RR_UNKNOWN; + g_access(cp, -1, 0, 0); + } if (fcp->acr != 0 || fcp->acw != 0 || fcp->ace != 0) { error = g_access(cp, fcp->acr, fcp->acw, fcp->ace); if (error != 0) { gctl_error(req, "Access request failed for %s", pp->name); - g_destroy_consumer(cp); - g_topology_unlock(); - return; - } - } - if (fcp->provider->sectorsize != pp->sectorsize) { - gctl_error(req, "Sector size doesn't fit for %s", - pp->name); - g_destroy_consumer(cp); - g_topology_unlock(); - return; - } - for (j = 0; j < sc->n_components; j++) { - if (strcmp(sc->components[j].gcons->provider->name, - pp->name) == 0) { - gctl_error(req, "Component %s already in %s", - pp->name, sc->geom->name); + g_detach(cp); g_destroy_consumer(cp); g_topology_unlock(); return; @@ -380,13 +408,15 @@ virstor_ctl_add(struct gctl_req *req, struct g_class *cp) sc->chunk_size; sc->components[nc].chunk_next = 0; sc->components[nc].chunk_reserved = 0; + sc->components[nc].flags = 0; + sc->components[nc].candelete = cd; + sc->components[nc].rotation_rate = rr; - if (sc->components[nc].chunk_count < 4) { - gctl_error(req, "Provider too small: %s", - cp->provider->name); - g_destroy_consumer(cp); - g_topology_unlock(); - return; + /* A provider underneath us doesn't support unmapped. */ + if ((pp->flags & G_PF_ACCEPT_UNMAPPED) == 0) { + LOG_MSG(LVL_INFO, "Cancelling unmapped because of %s", + pp->name); + sc->provider->flags &= ~G_PF_ACCEPT_UNMAPPED; } fill_metadata(sc, &md, nc, *hardcode); write_metadata(cp, &md); @@ -697,7 +727,7 @@ g_virstor_destroy_geom(struct gctl_req *req __unused, struct g_class *mp, "table for %s", sc->geom->name); count = 0; for (n = 0; n < sc->chunk_count; n++) { - if (sc->map[n].flags || VIRSTOR_MAP_ALLOCATED != 0) + if ((sc->map[n].flags & VIRSTOR_MAP_ALLOCATED) != 0) count++; } LOG_MSG(LVL_INFO, "Device %s has %d allocated chunks", @@ -956,6 +986,7 @@ virstor_geom_destroy(struct g_virstor_softc *sc, boolean_t force, } mtx_unlock(&sc->delayed_bio_q_mtx); mtx_destroy(&sc->delayed_bio_q_mtx); + mtx_destroy(&sc->completion_mtx); free(sc->map, M_GVIRSTOR); free(sc->components, M_GVIRSTOR); @@ -1104,6 +1135,8 @@ create_virstor_geom(struct g_class *mp, struct g_virstor_metadata *md) STAILQ_INIT(&sc->delayed_bio_q); mtx_init(&sc->delayed_bio_q_mtx, "gvirstor_delayed_bio_q_mtx", "gvirstor", MTX_DEF | MTX_RECURSE); + mtx_init(&sc->completion_mtx, "gvirstor_completion_mtx", "gvirstor", + MTX_DEF); sc->geom = gp; sc->provider = NULL; /* virstor_check_and_run will create it */ @@ -1138,6 +1171,7 @@ add_provider_to_geom(struct g_virstor_softc *sc, struct g_provider *pp, fcp = LIST_FIRST(&gp->consumer); cp = g_new_consumer(gp); + cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; error = g_attach(cp, pp); if (error != 0) { @@ -1145,12 +1179,27 @@ add_provider_to_geom(struct g_virstor_softc *sc, struct g_provider *pp, return (error); } + /* Cascade candelete and rotation rate. */ + component->candelete = 0; + component->rotation_rate = DISK_RR_UNKNOWN; + if (g_access(cp, 1, 0, 0) == 0) { + if (g_getattr("GEOM::candelete", cp, + &component->candelete) != 0) + component->candelete = 0; + if (g_getattr("GEOM::rotation_rate", cp, + &component->rotation_rate) != 0) + component->rotation_rate = DISK_RR_UNKNOWN; + g_access(cp, -1, 0, 0); + } + if (fcp != NULL) { if (fcp->provider->sectorsize != pp->sectorsize) { /* TODO: this can be made to work */ LOG_MSG(LVL_ERROR, "Provider %s of %s has invalid " "sector size (%d)", pp->name, sc->geom->name, pp->sectorsize); + g_detach(cp); + g_destroy_consumer(cp); return (EINVAL); } if (fcp->acr > 0 || fcp->acw || fcp->ace > 0) { @@ -1193,6 +1242,7 @@ virstor_check_and_run(struct g_virstor_softc *sc) size_t n, count; int index; int error; + bool unmapped; if (virstor_valid_components(sc) != sc->n_components) return; @@ -1221,8 +1271,19 @@ virstor_check_and_run(struct g_virstor_softc *sc) sc->map_sectors = sc->map_size / sc->sectorsize; count = 0; - for (n = 0; n < sc->n_components; n++) + unmapped = true; + for (n = 0; n < sc->n_components; n++) { + struct g_provider *cpp; + count += sc->components[n].chunk_count; + /* A provider underneath us doesn't support unmapped. */ + cpp = sc->components[n].gcons->provider; + if ((cpp->flags & G_PF_ACCEPT_UNMAPPED) == 0) { + LOG_MSG(LVL_INFO, "Cancelling unmapped because of %s", + cpp->name); + unmapped = false; + } + } LOG_MSG(LVL_INFO, "Device %s has %zu physical chunks and %zu virtual " "(%zu KB chunks)", sc->geom->name, count, sc->chunk_count, sc->chunk_size / 1024); @@ -1340,6 +1401,9 @@ virstor_check_and_run(struct g_virstor_softc *sc) sc->provider = g_new_providerf(sc->geom, "virstor/%s", sc->geom->name); + sc->provider->flags |= G_PF_DIRECT_SEND; + if (unmapped) + sc->provider->flags |= G_PF_ACCEPT_UNMAPPED; sc->provider->sectorsize = sc->sectorsize; sc->provider->mediasize = sc->virsize; g_error_provider(sc->provider, 0); @@ -1476,41 +1540,39 @@ g_virstor_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, indent, comp->chunk_reserved); sbuf_printf(sb, "%s<StorageFree>%u%%</StorageFree>\n", indent, - comp->chunk_next > 0 ? 100 - - ((comp->chunk_next + comp->chunk_reserved) * 100) / - comp->chunk_count : 100); + 100 - (u_int)((uint64_t)comp->chunk_next * 100 / + comp->chunk_count)); } else { /* For the whole thing */ - u_int count, used, i; - off_t size; + off_t count, used, size; + u_int pfree, i; count = used = size = 0; for (i = 0; i < sc->n_components; i++) { if (sc->components[i].gcons != NULL) { count += sc->components[i].chunk_count; - used += sc->components[i].chunk_next + - sc->components[i].chunk_reserved; + used += sc->components[i].chunk_next; size += sc->components[i].gcons-> provider->mediasize; } } + pfree = count > 0 ? 100 - (u_int)(used * 100 / count) : 100; sbuf_printf(sb, "%s<Status>" "Components=%u, Online=%u</Status>\n", indent, sc->n_components, virstor_valid_components(sc)); sbuf_printf(sb, "%s<State>%u%% physical free</State>\n", - indent, 100-(used * 100) / count); + indent, pfree); sbuf_printf(sb, "%s<ChunkSize>%zu</ChunkSize>\n", indent, sc->chunk_size); sbuf_printf(sb, "%s<PhysicalFree>%u%%</PhysicalFree>\n", - indent, used > 0 ? 100 - (used * 100) / count : 100); - sbuf_printf(sb, "%s<ChunkPhysicalCount>%u</ChunkPhysicalCount>\n", + indent, pfree); + sbuf_printf(sb, "%s<ChunkPhysicalCount>%jd</ChunkPhysicalCount>\n", indent, count); sbuf_printf(sb, "%s<ChunkVirtualCount>%zu</ChunkVirtualCount>\n", indent, sc->chunk_count); - sbuf_printf(sb, "%s<PhysicalBacking>%zu%%</PhysicalBacking>\n", - indent, - (count * 100) / sc->chunk_count); + sbuf_printf(sb, "%s<PhysicalBacking>%jd%%</PhysicalBacking>\n", + indent, count * 100 / (off_t)sc->chunk_count); sbuf_printf(sb, "%s<PhysicalBackingSize>%jd</PhysicalBackingSize>\n", indent, size); sbuf_printf(sb, "%s<VirtualSize>%jd</VirtualSize>\n", indent, @@ -1526,28 +1588,142 @@ g_virstor_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, static void g_virstor_done(struct bio *b) { + struct g_virstor_softc *sc; struct bio *parent_b; parent_b = b->bio_parent; + sc = parent_b->bio_to->geom->softc; if (b->bio_error != 0) { LOG_MSG(LVL_ERROR, "Error %d for offset=%ju, length=%ju, %s", b->bio_error, b->bio_offset, b->bio_length, b->bio_to->name); - if (parent_b->bio_error == 0) - parent_b->bio_error = b->bio_error; } + mtx_lock(&sc->completion_mtx); + if (parent_b->bio_error == 0) + parent_b->bio_error = b->bio_error; parent_b->bio_inbed++; parent_b->bio_completed += b->bio_completed; - if (parent_b->bio_children == parent_b->bio_inbed) { + mtx_unlock(&sc->completion_mtx); parent_b->bio_completed = parent_b->bio_length; g_io_deliver(parent_b, parent_b->bio_error); - } + } else + mtx_unlock(&sc->completion_mtx); g_destroy_bio(b); } +/* + * Pass the request down to every allocated component. A chunk may be + * allocated on any of them, so all have to be covered. Components without + * any allocated chunks never received a data write, so they are skipped. + */ +static void +g_virstor_passdown(struct g_virstor_softc *sc, struct bio *b) +{ + struct bio_queue_head bq; + struct g_virstor_component *comp; + struct bio *cb; + u_int n; + + bioq_init(&bq); + for (n = 0; n < sc->n_components; n++) { + comp = &sc->components[n]; + if (comp->gcons == NULL || + (comp->flags & VIRSTOR_PROVIDER_ALLOCATED) == 0) + continue; + cb = g_clone_bio(b); + if (cb == NULL) { + bioq_dismantle(&bq); + if (b->bio_error == 0) + b->bio_error = ENOMEM; + g_io_deliver(b, b->bio_error); + return; + } + cb->bio_done = g_virstor_done; + cb->bio_to = comp->gcons->provider; + cb->bio_caller1 = comp; + bioq_insert_tail(&bq, cb); + } + KASSERT(bioq_first(&bq) != NULL, + ("%s: No allocated components in %s", __func__, sc->geom->name)); + while ((cb = bioq_takefirst(&bq)) != NULL) { + comp = cb->bio_caller1; + cb->bio_caller1 = NULL; + LOG_REQ(LVL_MOREDEBUG, cb, "Firing request"); + g_io_request(cb, comp->gcons); + } +} + +/* + * Report BIO_DELETE support if any of the components supports it. + */ +static void +g_virstor_candelete(struct g_virstor_softc *sc, struct bio *b) +{ + int val; + u_int n; + + for (n = 0; n < sc->n_components; n++) { + if (sc->components[n].gcons != NULL && + sc->components[n].candelete) + break; + } + val = n < sc->n_components; + g_handleattr(b, "GEOM::candelete", &val, sizeof(val)); +} + +/* + * Report the rotation rate of the components, if they all agree on it. + */ +static void +g_virstor_rotation_rate(struct g_virstor_softc *sc, struct bio *b) +{ + uint16_t rr = DISK_RR_UNKNOWN; + bool first = true; + u_int n; + + for (n = 0; n < sc->n_components; n++) { + if (sc->components[n].gcons == NULL) + continue; + if (first) + rr = sc->components[n].rotation_rate; + else if (rr != sc->components[n].rotation_rate) { + rr = DISK_RR_UNKNOWN; + break; + } + first = false; + } + g_handleattr(b, "GEOM::rotation_rate", &rr, sizeof(rr)); +} + +/* + * Zero a part of the request's buffer. For unmapped requests addr is not a + * pointer, but an offset from the beginning of the buffer. + */ +static void +g_virstor_zero(struct bio *b, char *addr, size_t length) +{ + vm_page_t *ma; + size_t moff, len; + + if ((b->bio_flags & BIO_UNMAPPED) == 0) { + bzero(addr, length); + return; + } + moff = b->bio_ma_offset + (uintptr_t)addr; + ma = b->bio_ma + moff / PAGE_SIZE; + moff %= PAGE_SIZE; + while (length > 0) { + len = MIN(PAGE_SIZE - moff, length); + pmap_zero_page_area(*ma, moff, len); + ma++; + moff = 0; + length -= len; + } +} + /* * I/O starts here * Called in g_down thread @@ -1577,6 +1753,20 @@ g_virstor_start(struct bio *b) case BIO_WRITE: case BIO_DELETE: break; + case BIO_FLUSH: + g_virstor_passdown(sc, b); + return; + case BIO_GETATTR: + if (strcmp(b->bio_attribute, "GEOM::candelete") == 0) { + g_virstor_candelete(sc, b); + return; + } + if (strcmp(b->bio_attribute, "GEOM::rotation_rate") == 0) { + g_virstor_rotation_rate(sc, b); + return; + } + /* To which component should it be delivered? */ + /* FALLTHROUGH */ default: g_io_deliver(b, EOPNOTSUPP); return; @@ -1586,7 +1776,10 @@ g_virstor_start(struct bio *b) bioq_init(&bq); chunk_size = sc->chunk_size; - addr = b->bio_data; + if ((b->bio_flags & BIO_UNMAPPED) != 0) + addr = NULL; + else + addr = b->bio_data; offset = b->bio_offset; /* virtual offset and length */ length = b->bio_length; @@ -1608,7 +1801,8 @@ g_virstor_start(struct bio *b) /* Reads from unallocated chunks return zeroed * buffers */ if (b->bio_cmd == BIO_READ) - bzero(addr, in_chunk_length); + g_virstor_zero(b, addr, + in_chunk_length); } else { comp = &sc->components[me->provider_no]; @@ -1626,7 +1820,16 @@ g_virstor_start(struct bio *b) (off_t)me->provider_chunk * (off_t)chunk_size + in_chunk_offset; cb->bio_length = in_chunk_length; - cb->bio_data = addr; + if ((b->bio_flags & BIO_UNMAPPED) != 0) { + cb->bio_ma_offset += (uintptr_t)addr; + cb->bio_ma += cb->bio_ma_offset / + PAGE_SIZE; + cb->bio_ma_offset %= PAGE_SIZE; + cb->bio_ma_n = round_page( + cb->bio_ma_offset + + cb->bio_length) / PAGE_SIZE; + } else + cb->bio_data = addr; cb->bio_caller1 = comp; bioq_disksort(&bq, cb); } @@ -1716,6 +1919,7 @@ g_virstor_start(struct bio *b) cb->bio_done = g_virstor_done; cb->bio_offset = s_offset; cb->bio_data = (char *)data_me; + cb->bio_flags &= ~BIO_UNMAPPED; cb->bio_length = sc->sectorsize; cb->bio_caller1 = &sc->components[0]; bioq_disksort(&bq, cb); @@ -1736,11 +1940,19 @@ g_virstor_start(struct bio *b) cb->bio_offset = (off_t)me->provider_chunk*(off_t)chunk_size + in_chunk_offset; cb->bio_length = in_chunk_length; - cb->bio_data = addr; + if ((b->bio_flags & BIO_UNMAPPED) != 0) { + cb->bio_ma_offset += (uintptr_t)addr; + cb->bio_ma += cb->bio_ma_offset / PAGE_SIZE; + cb->bio_ma_offset %= PAGE_SIZE; + cb->bio_ma_n = round_page(cb->bio_ma_offset + + cb->bio_length) / PAGE_SIZE; + } else + cb->bio_data = addr; cb->bio_caller1 = comp; bioq_disksort(&bq, cb); } - addr += in_chunk_length; + if (b->bio_cmd != BIO_DELETE) + addr += in_chunk_length; length -= in_chunk_length; offset += in_chunk_length; } diff --git a/sys/geom/virstor/g_virstor.h b/sys/geom/virstor/g_virstor.h index a7f81b6bea9b..38e84612b3e0 100644 --- a/sys/geom/virstor/g_virstor.h +++ b/sys/geom/virstor/g_virstor.h @@ -76,6 +76,8 @@ struct g_virstor_component { unsigned int chunk_next; unsigned int chunk_reserved; unsigned int flags; + int candelete; + uint16_t rotation_rate; }; /* Internal geom instance data */ @@ -96,6 +98,7 @@ struct g_virstor_softc { size_t me_per_sector; /* # map entries in a sector */ STAILQ_HEAD(, g_virstor_bio_q) delayed_bio_q; /* Queue of delayed BIOs */ struct mtx delayed_bio_q_mtx; + struct mtx completion_mtx; /* Synchronizes completions */ }; /* "delayed BIOs" Queue element */
