This adds the page-pool logic to the dma-buf pools which allows
a pool to keep pre-allocated/flushed pages around which can
speed up allocation performance.
NOTE: The page-pools name is term preserved from ION, but it has
potential to be easily confused with dma-buf pools. Suggestions
for alternatives here would be great.
Cc: Laura Abbott <labb...@redhat.com>
Cc: Benjamin Gaignard <benjamin.gaign...@linaro.org>
Cc: Sumit Semwal <sumit.sem...@linaro.org>
Cc: Liam Mark <lm...@codeaurora.org>
Cc: Brian Starkey <brian.star...@arm.com>
Cc: Andrew F. Davis <a...@ti.com>
Cc: Chenbo Feng <fe...@google.com>
Cc: Alistair Strachan <astrac...@google.com>
Cc: dri-devel@lists.freedesktop.org
Signed-off-by: John Stultz <john.stu...@linaro.org>
---
drivers/dma-buf/pools/Makefile | 2 +-
drivers/dma-buf/pools/dmabuf-pools.h | 51 ++++++++++++
drivers/dma-buf/pools/page_pool.c | 157 +++++++++++++++++++++++++++++++++++
3 files changed, 209 insertions(+), 1 deletion(-)
create mode 100644 drivers/dma-buf/pools/page_pool.c
diff --git a/drivers/dma-buf/pools/Makefile b/drivers/dma-buf/pools/Makefile
index 6cb1284..a51ec25 100644
--- a/drivers/dma-buf/pools/Makefile
+++ b/drivers/dma-buf/pools/Makefile
@@ -1,2 +1,2 @@
# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_DMABUF_POOLS) += dmabuf-pools.o pool-ioctl.o
pool-helpers.o
+obj-$(CONFIG_DMABUF_POOLS) += dmabuf-pools.o pool-ioctl.o
pool-helpers.o page_pool.o
diff --git a/drivers/dma-buf/pools/dmabuf-pools.h
b/drivers/dma-buf/pools/dmabuf-pools.h
index 12110f2..e3a0aac 100644
--- a/drivers/dma-buf/pools/dmabuf-pools.h
+++ b/drivers/dma-buf/pools/dmabuf-pools.h
@@ -238,6 +238,57 @@ size_t dmabuf_pool_freelist_shrink(struct dmabuf_pool
*pool,
*/
size_t dmabuf_pool_freelist_size(struct dmabuf_pool *pool);
+/**
+ * functions for creating and destroying a page pool -- allows you
+ * to keep a page pool of pre allocated memory to use from your pool. Keeping
+ * a page pool of memory that is ready for dma, ie any cached mapping have been
+ * invalidated from the cache, provides a significant performance benefit on
+ * many systems
+ */
+
+/**
+ * struct dmabuf_page_pool - pagepool struct
+ * @high_count: number of highmem items in the pool
+ * @low_count: number of lowmem items in the pool
+ * @high_items: list of highmem items
+ * @low_items: list of lowmem items
+ * @mutex: lock protecting this struct and especially the count
+ * item list
+ * @gfp_mask: gfp_mask to use from alloc
+ * @order: order of pages in the pool
+ * @list: plist node for list of pools
+ *
+ * Allows you to keep a page pool of pre allocated pages to use from your pool.
+ * Keeping a pool of pages that is ready for dma, ie any cached mapping have
+ * been invalidated from the cache, provides a significant performance benefit
+ * on many systems
+ */
+struct dmabuf_page_pool {
+ int high_count;
+ int low_count;
+ struct list_head high_items;
+ struct list_head low_items;
+ struct mutex mutex;
+ gfp_t gfp_mask;
+ unsigned int order;
+ struct plist_node list;
+};
+
+struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask,
+ unsigned int order);
+void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool);
+struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool);
+void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page);
+
+/** dmabuf_page_pool_shrink - shrinks the size of the memory cached in the pool
+ * @pool: the page pool
+ * @gfp_mask: the memory type to reclaim
+ * @nr_to_scan: number of items to shrink in pages
+ *
+ * returns the number of items freed in pages
+ */
+int dmabuf_page_pool_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
+ int nr_to_scan);
long dmabuf_pool_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
diff --git a/drivers/dma-buf/pools/page_pool.c
b/drivers/dma-buf/pools/page_pool.c
new file mode 100644
index 0000000..c1fe994
--- /dev/null
+++ b/drivers/dma-buf/pools/page_pool.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * drivers/dma-buf/pools/page_pool.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019 Linaro Ltd.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+
+#include "dmabuf-pools.h"
+
+static inline struct page *dmabuf_page_pool_alloc_pages(
+ struct dmabuf_page_pool *pool)
+{
+ return alloc_pages(pool->gfp_mask, pool->order);
+}
+
+static void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool,
+ struct page *page)
+{
+ __free_pages(page, pool->order);
+}
+
+static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool,
+ struct page *page)
+{
+ mutex_lock(&pool->mutex);
+ if (PageHighMem(page)) {
+ list_add_tail(&page->lru, &pool->high_items);
+ pool->high_count++;
+ } else {
+ list_add_tail(&page->lru, &pool->low_items);
+ pool->low_count++;
+ }
+
+ mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
+ 1 << pool->order);
+ mutex_unlock(&pool->mutex);
+}
+
+static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool,
+ bool high)
+{
+ struct page *page;
+
+ if (high) {
+ WARN_ON(!pool->high_count);
+ page = list_first_entry(&pool->high_items, struct page, lru);
+ pool->high_count--;
+ } else {
+ WARN_ON(!pool->low_count);
+ page = list_first_entry(&pool->low_items, struct page, lru);
+ pool->low_count--;
+ }
+
+ list_del(&page->lru);
+ mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
+ -(1 << pool->order));
+ return page;
+}
+
+struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool)
+{
+ struct page *page = NULL;
+
+ WARN_ON(!pool);
+
+ mutex_lock(&pool->mutex);
+ if (pool->high_count)
+ page = dmabuf_page_pool_remove(pool, true);
+ else if (pool->low_count)
+ page = dmabuf_page_pool_remove(pool, false);
+ mutex_unlock(&pool->mutex);
+
+ if (!page)
+ page = dmabuf_page_pool_alloc_pages(pool);
+
+ return page;
+}
+
+void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page)
+{
+ WARN_ON(pool->order != compound_order(page));
+
+ dmabuf_page_pool_add(pool, page);
+}
+
+static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high)
+{
+ int count = pool->low_count;
+
+ if (high)
+ count += pool->high_count;
+
+ return count << pool->order;
+}
+
+int dmabuf_page_pool_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
+ int nr_to_scan)
+{
+ int freed = 0;
+ bool high;
+
+ if (current_is_kswapd())
+ high = true;
+ else
+ high = !!(gfp_mask & __GFP_HIGHMEM);
+
+ if (nr_to_scan == 0)
+ return dmabuf_page_pool_total(pool, high);
+
+ while (freed < nr_to_scan) {
+ struct page *page;
+
+ mutex_lock(&pool->mutex);
+ if (pool->low_count) {
+ page = dmabuf_page_pool_remove(pool, false);
+ } else if (high && pool->high_count) {
+ page = dmabuf_page_pool_remove(pool, true);
+ } else {
+ mutex_unlock(&pool->mutex);
+ break;
+ }
+ mutex_unlock(&pool->mutex);
+ dmabuf_page_pool_free_pages(pool, page);
+ freed += (1 << pool->order);
+ }
+
+ return freed;
+}
+
+struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask,
+ unsigned int order)
+{
+ struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+
+ if (!pool)
+ return NULL;
+ pool->high_count = 0;
+ pool->low_count = 0;
+ INIT_LIST_HEAD(&pool->low_items);
+ INIT_LIST_HEAD(&pool->high_items);
+ pool->gfp_mask = gfp_mask | __GFP_COMP;
+ pool->order = order;
+ mutex_init(&pool->mutex);
+ plist_node_init(&pool->list, order);
+
+ return pool;
+}
+
+void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool)
+{
+ kfree(pool);
+}
--
2.7.4
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
