This function copies the reply message directly into receiver's buffer (pool slice). Looking into the implementation closely, however, there are multiple occurrences of copy operations, kdbus_pool_slice_copy(), which then executes costly prolog/epilog functions, say set_fs() and write_begin/write_end(), on each entry/exit.
This patch uses a temporary buffer on the stack to compose a message and replace a costly function, kdbus_pool_slice_copy(), to memcpy() wherever possible, although I have to further confirm that this change is actually safe. Signed-off-by: AKASHI Takahiro <[email protected]> --- connection.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pool.c | 2 +- pool.h | 3 ++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/connection.c b/connection.c index 40cf133..d122be8 100644 --- a/connection.c +++ b/connection.c @@ -39,6 +39,10 @@ #include "policy.h" #include "util.h" +/* + * KDBUS_HACK1: optimize copy operations in kdbus_conn_queue_alloc() + */ + struct kdbus_conn_reply; /** @@ -232,7 +236,11 @@ exit_unref: static int kdbus_conn_payload_add(struct kdbus_conn *conn, struct kdbus_conn_queue *queue, const struct kdbus_kmsg *kmsg, +#if KDBUS_HACK1 + void *buf, size_t items, size_t vec_data) +#else size_t items, size_t vec_data) +#endif { const struct kdbus_item *item; int ret; @@ -253,8 +261,13 @@ static int kdbus_conn_payload_add(struct kdbus_conn *conn, KDBUS_ITEMS_SIZE(&kmsg->msg, items)) { switch (item->type) { case KDBUS_ITEM_PAYLOAD_VEC: { +#if KDBUS_HACK1 + u64 tmp[(KDBUS_ITEM_HEADER_SIZE + + sizeof(struct kdbus_vec) + 7)/8]; +#else char tmp[KDBUS_ITEM_HEADER_SIZE + sizeof(struct kdbus_vec)]; +#endif struct kdbus_item *it = (struct kdbus_item *)tmp; /* add item */ @@ -267,10 +280,14 @@ static int kdbus_conn_payload_add(struct kdbus_conn *conn, else it->vec.offset = ~0ULL; it->vec.size = item->vec.size; +#if KDBUS_HACK1 + memcpy(buf + items, it, (size_t)it->size); +#else ret = kdbus_pool_slice_copy(queue->slice, items, it, it->size); if (ret < 0) return ret; +#endif items += KDBUS_ALIGN8(it->size); /* \0-bytes record */ @@ -286,17 +303,28 @@ static int kdbus_conn_payload_add(struct kdbus_conn *conn, * null-bytes to the buffer which the \0-bytes * record would have shifted the alignment. */ +#if KDBUS_HACK1 + memset(buf + vec_data, 0, pad); +#else kdbus_pool_slice_copy(queue->slice, vec_data, "\0\0\0\0\0\0\0", pad); +#endif vec_data += pad; break; } /* copy kdbus_vec data from sender to receiver */ +#if KDBUS_HACK1 + if (copy_from_user(buf + vec_data, + KDBUS_PTR(item->vec.address), + (size_t)item->vec.size)) + return -EFAULT; +#else ret = kdbus_pool_slice_copy_user(queue->slice, vec_data, KDBUS_PTR(item->vec.address), item->vec.size); if (ret < 0) return ret; +#endif vec_data += item->vec.size; break; @@ -314,10 +342,14 @@ static int kdbus_conn_payload_add(struct kdbus_conn *conn, it->size = sizeof(tmp); it->memfd.size = item->memfd.size; it->memfd.fd = -1; +#if KDBUS_HACK1 + memcpy(buf + items, it, (size_t)it->size); +#else ret = kdbus_pool_slice_copy(queue->slice, items, it, it->size); if (ret < 0) return ret; +#endif /* grab reference of incoming file */ ret = kdbus_conn_memfd_ref(item, &fp); @@ -459,7 +491,13 @@ static int kdbus_conn_queue_alloc(struct kdbus_conn *conn, size_t fds = 0; size_t meta = 0; size_t vec_data; +#if KDBUS_HACK1 + size_t want; + char *tmp; + struct kdbus_item *it; +#else size_t want, have; +#endif int ret = 0; BUG_ON(!mutex_is_locked(&conn->lock)); @@ -518,6 +556,13 @@ static int kdbus_conn_queue_alloc(struct kdbus_conn *conn, /* do not give out more than half of the remaining space */ want = vec_data + kmsg->vecs_size; +#if KDBUS_HACK1 + ret = kdbus_pool_slice_alloc(conn->pool, &queue->slice, want); + if (ret < 0) { + ret = -EXFULL; + goto exit; + } +#else have = kdbus_pool_remain(conn->pool); if (want < have && want > have / 2) { ret = -EXFULL; @@ -528,7 +573,59 @@ static int kdbus_conn_queue_alloc(struct kdbus_conn *conn, ret = kdbus_pool_slice_alloc(conn->pool, &queue->slice, want); if (ret < 0) goto exit; +#endif + +#if KDBUS_HACK1 + tmp = kmalloc(vec_data, GFP_KERNEL); + + /* copy the message header */ + memcpy(tmp, &kmsg->msg, size); + /* update the size */ + *(size_t *)tmp = msg_size; + if (dst_name_len > 0) { + it = (struct kdbus_item *)&tmp[size]; + it->size = KDBUS_ITEM_HEADER_SIZE + dst_name_len; + it->type = KDBUS_ITEM_DST_NAME; + memcpy(it->str, kmsg->dst_name, dst_name_len); + } + + /* add PAYLOAD items */ + if (payloads > 0) { + ret = kdbus_conn_payload_add(conn, queue, kmsg, + tmp, payloads, vec_data); + if (ret < 0) + goto exit_pool_free; + } + + /* add a FDS item; the array content will be updated at RECV time */ + if (kmsg->fds_count > 0) { + it = (struct kdbus_item *)&tmp[fds]; + it->type = KDBUS_ITEM_FDS; + it->size = KDBUS_ITEM_HEADER_SIZE + + (kmsg->fds_count * sizeof(int)); + memcpy(tmp + fds, it, KDBUS_ITEM_HEADER_SIZE); + + ret = kdbus_conn_fds_ref(queue, kmsg->fds, kmsg->fds_count); + if (ret < 0) + goto exit_pool_free; + + /* remember the array to update at RECV */ + queue->fds = fds + offsetof(struct kdbus_item, fds); + queue->fds_count = kmsg->fds_count; + } + + /* append message metadata/credential items */ + if (meta > 0) { + memcpy(tmp + meta, kmsg->meta->data, kmsg->meta->size); + } + + /* finally */ + ret = kdbus_pool_copy(queue->slice, 0, tmp, NULL, 0, want); + kfree(tmp); + if (ret < 0) + goto exit_pool_free; +#else /* copy the message header */ ret = kdbus_pool_slice_copy(queue->slice, 0, &kmsg->msg, size); if (ret < 0) @@ -591,6 +688,7 @@ static int kdbus_conn_queue_alloc(struct kdbus_conn *conn, if (ret < 0) goto exit_pool_free; } +#endif queue->priority = kmsg->msg.priority; *q = queue; diff --git a/pool.c b/pool.c index 4267c54..fcc54af 100644 --- a/pool.c +++ b/pool.c @@ -462,7 +462,7 @@ static int kdbus_pool_copy_data(struct page *p, size_t start, } /* copy data to the receiver's pool */ -static size_t kdbus_pool_copy(const struct kdbus_pool_slice *slice, size_t off, +size_t kdbus_pool_copy(const struct kdbus_pool_slice *slice, size_t off, const void __user *data, struct file *f_src, size_t off_src, size_t len) { diff --git a/pool.h b/pool.h index d1718eb..5b89d31 100644 --- a/pool.h +++ b/pool.h @@ -30,6 +30,9 @@ void kdbus_pool_slice_free(struct kdbus_pool_slice *slice); struct kdbus_pool_slice *kdbus_pool_slice_find(struct kdbus_pool *pool, size_t off); size_t kdbus_pool_slice_offset(const struct kdbus_pool_slice *slice); +size_t kdbus_pool_copy(const struct kdbus_pool_slice *slice, size_t off, + const void __user *data, struct file *f_src, + size_t off_src, size_t len); ssize_t kdbus_pool_slice_copy(const struct kdbus_pool_slice *slice, size_t off, const void *data, size_t len); ssize_t -- 1.7.9.5 _______________________________________________ systemd-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/systemd-devel
