From: Thierry Reding <[email protected]> This API can be used to allocate a number of CMA pages starting at a fixed offset. This is useful, for example, if the CMA area is used as backing storage for a nested allocator that has stricter requirements than CMA itself.
Suggested-by: Marek Szyprowski <[email protected]> Signed-off-by: Thierry Reding <[email protected]> --- include/linux/cma.h | 4 ++ include/trace/events/cma.h | 63 +++++++++++++++++++ mm/cma.c | 148 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/include/linux/cma.h b/include/linux/cma.h index 8555d38a97b1..844404459a42 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -49,11 +49,15 @@ extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, struct cma **res_cma); extern struct page *cma_alloc(struct cma *cma, unsigned long count, unsigned int align, bool no_warn); +extern struct page *cma_alloc_at(struct cma *cma, unsigned long offset, + unsigned long count, bool no_warn); extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count); struct page *cma_alloc_frozen(struct cma *cma, unsigned long count, unsigned int align, bool no_warn); struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order); +struct page *cma_alloc_at_frozen(struct cma *cma, unsigned long offset, + unsigned long count, bool no_warn); bool cma_release_frozen(struct cma *cma, const struct page *pages, unsigned long count); diff --git a/include/trace/events/cma.h b/include/trace/events/cma.h index 37195edf2498..00b622a9da97 100644 --- a/include/trace/events/cma.h +++ b/include/trace/events/cma.h @@ -132,6 +132,69 @@ TRACE_EVENT(cma_alloc_busy_retry, __entry->align) ); +TRACE_EVENT(cma_alloc_at_start, + + TP_PROTO(const char *name, unsigned long pfn, + unsigned long request_count, unsigned long available_count, + unsigned long total_count), + + TP_ARGS(name, pfn, request_count, available_count, total_count), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, pfn) + __field(unsigned long, request_count) + __field(unsigned long, available_count) + __field(unsigned long, total_count) + ), + + TP_fast_assign( + __assign_str(name); + __entry->pfn = pfn; + __entry->request_count = request_count; + __entry->available_count = available_count; + __entry->total_count = total_count; + ), + + TP_printk("name=%s pfn=%lx, request_count=%lu available_count=%lu total_count=%lu", + __get_str(name), + __entry->pfn, + __entry->request_count, + __entry->available_count, + __entry->total_count) +); + +TRACE_EVENT(cma_alloc_at_finish, + + TP_PROTO(const char *name, unsigned long pfn, const struct page *page, + unsigned long count, int errorno), + + TP_ARGS(name, pfn, page, count, errorno), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, pfn) + __field(const struct page *, page) + __field(unsigned long, count) + __field(int, errorno) + ), + + TP_fast_assign( + __assign_str(name); + __entry->pfn = pfn; + __entry->page = page; + __entry->count = count; + __entry->errorno = errorno; + ), + + TP_printk("name=%s pfn=0x%lx page=%p count=%lu errorno=%d", + __get_str(name), + __entry->pfn, + __entry->page, + __entry->count, + __entry->errorno) +); + #endif /* _TRACE_CMA_H */ /* This part must be outside protection */ diff --git a/mm/cma.c b/mm/cma.c index a10ea37a261d..1e1ebae79090 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -936,6 +936,141 @@ struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order) return __cma_alloc_frozen(cma, 1 << order, order, gfp); } +static int cma_range_alloc_at(struct cma *cma, struct cma_memrange *cmr, + unsigned long offset, unsigned long count, + struct page **pagep, gfp_t gfp) +{ + struct page *page = NULL; + unsigned long pfn; + int ret = -EBUSY; + + spin_lock_irq(&cma->lock); + + /* + * If the request is larger than the available number of pages, stop + * right away. + */ + if (count > cma->available_count) + goto unlock; + + ret = bitmap_allocate(cmr->bitmap, offset, count); + if (ret < 0) + goto unlock; + + pfn = cmr->base_pfn + offset; + page = pfn_to_page(pfn); + + /* + * Do not hand out page ranges that are not contiguous, so + * callers can just iterate the pages without having to worry + * about these corner cases. + */ + if (!page_range_contiguous(page, count)) { + pr_warn_ratelimited("%s: %s: skipping non-contiguous area [0x%lx-0x%lx]", + __func__, cma->name, pfn, pfn + count - 1); + ret = -EBUSY; + goto clear; + } + + cma->available_count -= count; + + /* + * It's safe to drop the lock here. We've marked this region for + * our exclusive use. If the migration fails we will take the + * lock again and unmark it. + */ + spin_unlock_irq(&cma->lock); + + mutex_lock(&cma->alloc_mutex); + ret = alloc_contig_frozen_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp); + mutex_unlock(&cma->alloc_mutex); + + if (ret < 0) + goto free; + + *pagep = page; + + return 0; + +free: + /* we need to reacquire the lock to clean up the internal state */ + spin_lock_irq(&cma->lock); + cma->available_count += count; +clear: + bitmap_clear(cmr->bitmap, offset, count); +unlock: + spin_unlock_irq(&cma->lock); + return ret; +} + +static struct page *__cma_alloc_at_frozen(struct cma *cma, unsigned long offset, + unsigned long count, gfp_t gfp) +{ + const char *name = cma ? cma->name : NULL; + struct page *page = NULL; + int ret = -ENOMEM, r; + unsigned long i; + + if (!cma || !cma->count) + return page; + + pr_debug("%s(cma %p, name: %s, offset %lu, count %lu)\n", __func__, + (void *)cma, cma->name, offset, count); + + if (!count) + return page; + + trace_cma_alloc_at_start(name, offset, count, cma->available_count, + cma->count); + + for (r = 0; r < cma->nranges; r++) { + page = NULL; + + ret = cma_range_alloc_at(cma, &cma->ranges[r], offset, count, + &page, gfp); + if (ret != -EBUSY || page) + break; + } + + /* + * CMA can allocate multiple page blocks, which results in different + * blocks being marked with different tags. Reset the tags to ignore + * those page blocks. + */ + if (page) { + for (i = 0; i < count; i++) + page_kasan_tag_reset(page + i); + } + + if (ret && !(gfp & __GFP_NOWARN)) { + pr_err_ratelimited("%s: %s: alloc failed, request: %lu, %lu pages, ret: %d\n", + __func__, cma->name, offset, count, ret); + cma_debug_show_areas(cma); + } + + pr_debug("%s(): returned %p\n", __func__, page); + trace_cma_alloc_at_finish(name, page ? page_to_pfn(page) : 0, page, + count, ret); + + if (page) { + count_vm_event(CMA_ALLOC_SUCCESS); + cma_sysfs_account_success_pages(cma, count); + } else { + count_vm_event(CMA_ALLOC_FAIL); + cma_sysfs_account_fail_pages(cma, count); + } + + return page; +} + +struct page *cma_alloc_at_frozen(struct cma *cma, unsigned long offset, + unsigned long count, bool no_warn) +{ + gfp_t gfp = GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0); + + return __cma_alloc_at_frozen(cma, offset, count, gfp); +} + /** * cma_alloc() - allocate pages from contiguous area * @cma: Contiguous memory region for which the allocation is performed. @@ -959,6 +1094,19 @@ struct page *cma_alloc(struct cma *cma, unsigned long count, } EXPORT_SYMBOL_GPL(cma_alloc); +struct page *cma_alloc_at(struct cma *cma, unsigned long pfn, + unsigned long count, bool no_warn) +{ + struct page *page; + + page = cma_alloc_at_frozen(cma, pfn, count, no_warn); + if (page) + set_pages_refcounted(page, count); + + return page; +} +EXPORT_SYMBOL_GPL(cma_alloc_at); + static struct cma_memrange *find_cma_memrange(struct cma *cma, const struct page *pages, unsigned long count) { -- 2.55.0
