Re: [PATCH 1/3 v2] chunkd: remove sendfile(2) zero-copy support
On 07/18/2010 07:10 PM, Jeff Garzik wrote: On 07/17/2010 11:45 PM, Steven Dake wrote: On 07/16/2010 10:46 PM, Jeff Garzik wrote: chunkd: remove sendfile(2) zero-copy support chunkd will be soon checksumming data in main memory. That removes the utility of a zero-copy interface which bypasses the on-heap data requirement. Signed-off-by: Jeff Garzik May be able to use vmsplice with sendfile (if linux is only target platform). Haven't tried it myself, but the operations look interesting at achieving zero copy with sockets from memory addresses. As an aside, Project Hail -is- intended to be portable to other operating systems. That said, I happily use OS-specific features if they have a measurable impact on our core code paths. Another OS-specific feature I plan on using, for example, is sync_file_range(2) for large objects. We can make use of the technique used by MythTV for streaming, which Linus describes here: http://marc.info/?l=linux-kernel&m=127429771726842&w=2 http://marc.info/?l=linux-kernel&m=127431438118461&w=2 -- To unsubscribe from this list: send the line "unsubscribe hail-devel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/3 v2] chunkd: remove sendfile(2) zero-copy support
On 07/17/2010 11:45 PM, Steven Dake wrote: On 07/16/2010 10:46 PM, Jeff Garzik wrote: chunkd: remove sendfile(2) zero-copy support chunkd will be soon checksumming data in main memory. That removes the utility of a zero-copy interface which bypasses the on-heap data requirement. Signed-off-by: Jeff Garzik May be able to use vmsplice with sendfile (if linux is only target platform). Haven't tried it myself, but the operations look interesting at achieving zero copy with sockets from memory addresses. Even though the man pages say "only for pipes", this syscall definitely works with TCP. The big question: is it actually faster than read()+write() ? Years ago, I experimented with using some fancy new Linux-specific syscalls in a from-scratch implementation of cp(1). It turned out that read()+write() was faster than other methods. That was file->file copying. It's probably worth investigating vmsplice() for our file->checksum->TCP case, definitely. Jeff -- To unsubscribe from this list: send the line "unsubscribe hail-devel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/3 v2] chunkd: remove sendfile(2) zero-copy support
On 07/16/2010 10:46 PM, Jeff Garzik wrote:
chunkd/be-fs.c | 60
chunkd/chunkd.h | 14 -
chunkd/object.c | 31
chunkd/server.c | 28 --
configure.ac|3 --
5 files changed, 15 insertions(+), 121 deletions(-)
commit e87d06526fa2052a1cecbed65ec2fac263c5e7d8
Author: Jeff Garzik
Date: Sat Jul 17 00:26:41 2010 -0400
chunkd: remove sendfile(2) zero-copy support
chunkd will be soon checksumming data in main memory. That removes
the utility of a zero-copy interface which bypasses the on-heap
data requirement.
Signed-off-by: Jeff Garzik
diff --git a/chunkd/be-fs.c b/chunkd/be-fs.c
index 4b851a7..0a81134 100644
--- a/chunkd/be-fs.c
+++ b/chunkd/be-fs.c
@@ -25,9 +25,6 @@
#include
#include
#include
-#if defined(HAVE_SYS_SENDFILE_H)
-#include
-#endif
#include
#include
#include
@@ -52,7 +49,6 @@ struct fs_obj {
int in_fd;
char*in_fn;
- off_t sendfile_ofs;
};
struct be_fs_obj_hdr {
@@ -547,62 +543,6 @@ ssize_t fs_obj_write(struct backend_obj *bo, const void
*ptr, size_t len)
return rc;
}
-#if defined(HAVE_SENDFILE)&& defined(__linux__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- struct fs_obj *obj = bo->private;
- ssize_t rc;
-
- if (obj->sendfile_ofs == 0) {
- obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
- obj->sendfile_ofs += bo->key_len;
- }
-
- rc = sendfile(out_fd, obj->in_fd,&obj->sendfile_ofs, len);
- if (rc< 0)
- applog(LOG_ERR, "obj sendfile(%s) failed: %s",
- obj->in_fn, strerror(errno));
-
- return rc;
-}
-
-#elif defined(HAVE_SENDFILE)&& defined(__FreeBSD__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- struct fs_obj *obj = bo->private;
- ssize_t rc;
- off_t sbytes = 0;
-
- if (obj->sendfile_ofs == 0) {
- obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
- obj->sendfile_ofs += bo->key_len;
- }
-
- rc = sendfile(obj->in_fd, out_fd, obj->sendfile_ofs, len,
- NULL,&sbytes, 0);
- if (rc< 0) {
- applog(LOG_ERR, "obj sendfile(%s) failed: %s",
- obj->in_fn, strerror(errno));
- return rc;
- }
-
- obj->sendfile_ofs += sbytes;
-
- return sbytes;
-}
-
-#else
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- applog(LOG_ERR, "BUG: sendfile used but not supported");
- return -EOPNOTSUPP;
-}
-
-#endif /* HAVE_SENDFILE&& HAVE_SYS_SENDFILE_H */
-
bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
unsigned char *md, bool sync_data)
{
diff --git a/chunkd/chunkd.h b/chunkd/chunkd.h
index 72833f7..716704b 100644
--- a/chunkd/chunkd.h
+++ b/chunkd/chunkd.h
@@ -39,8 +39,6 @@ enum {
CLI_DATA_BUF_SZ = 16 * 1024,
CHD_TRASH_MAX = 1000,
-
- CLI_MAX_SENDFILE_SZ = 512 * 1024,
};
struct client;
@@ -54,7 +52,6 @@ struct client_write {
uint64_tlen;/* write buffer length */
cli_write_func cb; /* callback */
void*cb_data; /* data passed to cb */
- boolsendfile; /* using sendfile? */
struct list_headnode;
};
@@ -268,7 +265,6 @@ extern bool fs_obj_delete(uint32_t table_id, const char
*user,
const void *kbuf, size_t klen,
enum chunk_errcode *err_code);
extern int fs_obj_disable(const char *fn);
-extern ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len);
extern int fs_list_objs_open(struct fs_obj_lister *t,
const char *root_path, uint32_t table_id);
extern int fs_list_objs_next(struct fs_obj_lister *t, char **fnp);
@@ -323,7 +319,6 @@ extern void applog(int prio, const char *fmt, ...);
extern bool cli_err(struct client *cli, enum chunk_errcode code, bool
recycle_ok);
extern int cli_writeq(struct client *cli, const void *buf, unsigned int
buflen,
cli_write_func cb, void *cb_data);
-extern bool cli_wr_sendfile(struct client *, cli_write_func);
extern bool cli_rd_set_poll(struct client *cli, bool readable);
extern void cli_wr_set_poll(struct client *cli, bool writable);
extern bool cli_cb_free(struct client *cli, struct client_write *wr,
@@ -342,15 +337,6 @@ extern void read_config(void);
/* selfcheck.c */
extern int chk_spawn(TCHDB *hdb);
-static inline bool use_sendfile(struct client *cli)
-{
-#if defined(HAVE_SENDFILE)&& defined(HAVE_SYS_SENDFILE_H)
- return cli->ssl ? false : true;
-#else
-
[PATCH 1/3 v2] chunkd: remove sendfile(2) zero-copy support
chunkd/be-fs.c | 60
chunkd/chunkd.h | 14 -
chunkd/object.c | 31
chunkd/server.c | 28 --
configure.ac|3 --
5 files changed, 15 insertions(+), 121 deletions(-)
commit e87d06526fa2052a1cecbed65ec2fac263c5e7d8
Author: Jeff Garzik
Date: Sat Jul 17 00:26:41 2010 -0400
chunkd: remove sendfile(2) zero-copy support
chunkd will be soon checksumming data in main memory. That removes
the utility of a zero-copy interface which bypasses the on-heap
data requirement.
Signed-off-by: Jeff Garzik
diff --git a/chunkd/be-fs.c b/chunkd/be-fs.c
index 4b851a7..0a81134 100644
--- a/chunkd/be-fs.c
+++ b/chunkd/be-fs.c
@@ -25,9 +25,6 @@
#include
#include
#include
-#if defined(HAVE_SYS_SENDFILE_H)
-#include
-#endif
#include
#include
#include
@@ -52,7 +49,6 @@ struct fs_obj {
int in_fd;
char*in_fn;
- off_t sendfile_ofs;
};
struct be_fs_obj_hdr {
@@ -547,62 +543,6 @@ ssize_t fs_obj_write(struct backend_obj *bo, const void
*ptr, size_t len)
return rc;
}
-#if defined(HAVE_SENDFILE) && defined(__linux__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- struct fs_obj *obj = bo->private;
- ssize_t rc;
-
- if (obj->sendfile_ofs == 0) {
- obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
- obj->sendfile_ofs += bo->key_len;
- }
-
- rc = sendfile(out_fd, obj->in_fd, &obj->sendfile_ofs, len);
- if (rc < 0)
- applog(LOG_ERR, "obj sendfile(%s) failed: %s",
- obj->in_fn, strerror(errno));
-
- return rc;
-}
-
-#elif defined(HAVE_SENDFILE) && defined(__FreeBSD__)
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- struct fs_obj *obj = bo->private;
- ssize_t rc;
- off_t sbytes = 0;
-
- if (obj->sendfile_ofs == 0) {
- obj->sendfile_ofs += sizeof(struct be_fs_obj_hdr);
- obj->sendfile_ofs += bo->key_len;
- }
-
- rc = sendfile(obj->in_fd, out_fd, obj->sendfile_ofs, len,
- NULL, &sbytes, 0);
- if (rc < 0) {
- applog(LOG_ERR, "obj sendfile(%s) failed: %s",
- obj->in_fn, strerror(errno));
- return rc;
- }
-
- obj->sendfile_ofs += sbytes;
-
- return sbytes;
-}
-
-#else
-
-ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len)
-{
- applog(LOG_ERR, "BUG: sendfile used but not supported");
- return -EOPNOTSUPP;
-}
-
-#endif /* HAVE_SENDFILE && HAVE_SYS_SENDFILE_H */
-
bool fs_obj_write_commit(struct backend_obj *bo, const char *user,
unsigned char *md, bool sync_data)
{
diff --git a/chunkd/chunkd.h b/chunkd/chunkd.h
index 72833f7..716704b 100644
--- a/chunkd/chunkd.h
+++ b/chunkd/chunkd.h
@@ -39,8 +39,6 @@ enum {
CLI_DATA_BUF_SZ = 16 * 1024,
CHD_TRASH_MAX = 1000,
-
- CLI_MAX_SENDFILE_SZ = 512 * 1024,
};
struct client;
@@ -54,7 +52,6 @@ struct client_write {
uint64_tlen;/* write buffer length */
cli_write_func cb; /* callback */
void*cb_data; /* data passed to cb */
- boolsendfile; /* using sendfile? */
struct list_headnode;
};
@@ -268,7 +265,6 @@ extern bool fs_obj_delete(uint32_t table_id, const char
*user,
const void *kbuf, size_t klen,
enum chunk_errcode *err_code);
extern int fs_obj_disable(const char *fn);
-extern ssize_t fs_obj_sendfile(struct backend_obj *bo, int out_fd, size_t len);
extern int fs_list_objs_open(struct fs_obj_lister *t,
const char *root_path, uint32_t table_id);
extern int fs_list_objs_next(struct fs_obj_lister *t, char **fnp);
@@ -323,7 +319,6 @@ extern void applog(int prio, const char *fmt, ...);
extern bool cli_err(struct client *cli, enum chunk_errcode code, bool
recycle_ok);
extern int cli_writeq(struct client *cli, const void *buf, unsigned int buflen,
cli_write_func cb, void *cb_data);
-extern bool cli_wr_sendfile(struct client *, cli_write_func);
extern bool cli_rd_set_poll(struct client *cli, bool readable);
extern void cli_wr_set_poll(struct client *cli, bool writable);
extern bool cli_cb_free(struct client *cli, struct client_write *wr,
@@ -342,15 +337,6 @@ extern void read_config(void);
/* selfcheck.c */
extern int chk_spawn(TCHDB *hdb);
-static inline bool use_sendfile(struct client *cli)
-{
-#if defined(HAVE_SENDFILE) && defined(HAVE_SYS_SENDFILE_H)
- return cli->ssl ? false : true;
-#else
- return false;
-#endif
-}
-
#ifndef HAVE_STRNLEN
e
