This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 337e689711e5a610ec00ad4fcaede162681bef55 Author: daimin <[email protected]> AuthorDate: Tue Jan 25 13:02:37 2022 +0800 HDFS-16403. Improve FUSE IO performance by supporting FUSE parameter max_background (#3842) Reviewed-by: Istvan Fajth <[email protected]> Reviewed-by: Wei-Chiu Chuang <[email protected]> (cherry picked from commit d69938994eec9c835906d28b0ae755dfd6698dcd) (cherry picked from commit f4241834cf4bf3cdc43a9c012a4d526b0299d88e) --- .../src/main/native/fuse-dfs/doc/README | 1 + .../src/main/native/fuse-dfs/fuse_dfs.c | 6 ++++++ .../src/main/native/fuse-dfs/fuse_init.c | 4 ++-- .../src/main/native/fuse-dfs/fuse_options.c | 11 +++++++---- .../src/main/native/fuse-dfs/fuse_options.h | 1 + .../src/main/native/fuse-dfs/test/TestFuseDFS.java | 1 + 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README index e8cc0e5..0dc1721 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README @@ -88,6 +88,7 @@ rw -odebug (do not daemonize - aka -d in fuse speak) -obig_writes (use fuse big_writes option so as to allow better performance of writes on kernels >= 2.6.26) -initchecks - have fuse-dfs try to connect to hdfs to ensure all is ok upon startup. recommended to have this on +-omax_background=%d (maximum number of pending "background" requests - see fuse docs) The defaults are: entry,attribute_timeouts = 60 seconds diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c index f693032..b9b0100 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c @@ -81,6 +81,7 @@ int main(int argc, char *argv[]) options.rdbuffer_size = 10*1024*1024; options.attribute_timeout = 60; options.entry_timeout = 60; + options.max_background = 0; if (-1 == fuse_opt_parse(&args, &options, dfs_opts, dfs_options)) { return -1; @@ -114,6 +115,11 @@ int main(int argc, char *argv[]) snprintf(buf, sizeof buf, "-oentry_timeout=%d",options.entry_timeout); fuse_opt_add_arg(&args, buf); + + if (options.max_background > 0) { + snprintf(buf, sizeof buf, "-omax_background=%d",options.max_background); + fuse_opt_add_arg(&args, buf); + } } if (options.nn_uri == NULL) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c index 4da6da0..9799c24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c @@ -91,11 +91,11 @@ static void dfsPrintOptions(FILE *fp, const struct options *o) INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, " "debug=%d, read_only=%d, initchecks=%d, " "no_permissions=%d, usetrash=%d, entry_timeout=%d, " - "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]", + "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d, max_background=%d ]", (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, o->debug, o->read_only, o->initchecks, o->no_permissions, o->usetrash, o->entry_timeout, - o->attribute_timeout, o->rdbuffer_size, o->direct_io); + o->attribute_timeout, o->rdbuffer_size, o->direct_io, o->max_background); } void *dfs_init(struct fuse_conn_info *conn) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c index 8461ce4..b4082c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c @@ -37,11 +37,13 @@ void print_options() { "\tentry_timeout=%d\n" "\tattribute_timeout=%d\n" "\tprivate=%d\n" - "\trdbuffer_size=%d (KBs)\n", - options.protected, options.nn_uri, options.nn_port, options.debug, + "\trdbuffer_size=%d (KBs)\n" + "\tmax_background=%d\n", + options.protected, options.nn_uri, options.nn_port, options.debug, options.read_only, options.usetrash, options.entry_timeout, options.attribute_timeout, options.private, - (int)options.rdbuffer_size / 1024); + (int)options.rdbuffer_size / 1024, + options.max_background); } const char *program; @@ -56,7 +58,7 @@ void print_usage(const char *pname) "[-ousetrash] [-obig_writes] [-oprivate (single user)] [ro] " "[-oserver=<hadoop_servername>] [-oport=<hadoop_port>] " "[-oentry_timeout=<secs>] [-oattribute_timeout=<secs>] " - "[-odirect_io] [-onopoermissions] [-o<other fuse option>] " + "[-odirect_io] [-onopoermissions] [-omax_background=<size>] [-o<other fuse option>] " "<mntpoint> [fuse options]\n", pname); printf("NOTE: debugging option for fuse is -debug\n"); } @@ -87,6 +89,7 @@ struct fuse_opt dfs_opts[] = DFSFS_OPT_KEY("protected=%s", protected, 0), DFSFS_OPT_KEY("port=%d", nn_port, 0), DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0), + DFSFS_OPT_KEY("max_background=%d", max_background, 0), FUSE_OPT_KEY("private", KEY_PRIVATE), FUSE_OPT_KEY("ro", KEY_RO), diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h index 4bfc235..2d00f1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h @@ -34,6 +34,7 @@ struct options { int private; size_t rdbuffer_size; int direct_io; + int max_background; } options; extern struct fuse_opt dfs_opts[]; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java index dabbe00..ec5c4e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java @@ -187,6 +187,7 @@ public class TestFuseDFS { "-ononempty", // Don't complain about junk in mount point "-f", // Don't background the process "-ordbuffer=32768", // Read buffer size in kb + "-omax_background=100", // Set fuse max_background=100 (12 by default) "rw" }; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
