Not seen this myself but you should update to at least CentOS 7.3, ideally 7.4. I believe a lot of cephfs fixes went into those kernels. If you still have the issue with the CentOS kernels, test with the latest upstream kernel. And/or test with latest Fuse client.
On Tue, Dec 5, 2017 at 12:01 PM, 张建 <[email protected]> wrote: > Hello, > > My problem description: > On cephfs client 1: > Copy a directory "base" which contains hundreds of rpm files to the > mounted cephfs directory. > then , change to cephfs directory and execute "ls -l base", it returns > quickly. This is no problem. > > On the other cephfs client, > Change to the mounted cephfs directory and execute "ls -l base", it blocks > for long time. > Execute "strace ls -l base", we can find it blocks in lstat files (about 5 > seconds blocking for each file). > > Is there anybody met this problem? > > ========================================================== > cephfs client information: > OS: CentOS Linux release 7.2.1511 (Core) > kernel: Linux 3.10.0-327.el7.x86_64 > mount type: with ceph kernel drvier > > ========================================================== > ceph cluster information: > OS: CentOS Linux release 7.2.1511 (Core) > kernel: Linux 3.10.0-327.el7.x86_64 > ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous > (stable) > ceph cluster node1: 1 mon, 1 mgr, 1 mds, 4 osd > ceph cluster node2: 1 mon, 1 mgr, 1 mds, 4 osd > ceph cluster node3: 1 mon, 1 mgr, 1 mds, 4 osd > Each osd is deployed with bluestore type (4TB HDD block device + 240GB SSD > block.db device). > public network: port 1 of 56Gb/s Dual Port Infiniband, IPoIB > cluster network: port 2 of 56Gb/s Dual Port Infiniband, IPoIB > > ceph.conf: > ------------------------- > [global] > fsid = 86887b08-6ff4-4ec1-a622-64bc688d1f2f > mon_initial_members = storage-0, storage-1, storage-2 > mon_host = 10.0.30.11,10.0.30.12,10.0.30.13 > auth_cluster_required = cephx > auth_service_required = cephx > auth_client_required = cephx > > public_network = 10.0.30.0/24 > cluster_network = 10.0.40.0/24 > > osd_pool_default_size = 2 > osd_pool_default_min_size = 1 > > osd_objectstore = bluestore > bluestore_block_db_size = 236223201280 > > ========================================================== > strace log: > ------------------------- > [root@computing-1 cephfs]# strace ls -l base > execve("/usr/bin/ls", ["ls", "-l", "base"], [/* 24 vars */]) = 0 > brk(0) = 0x155f000 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b91000 > access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or > directory) > open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0 > mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b83000 > close(3) = 0 > open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3 > read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300j\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=155744, ...}) = 0 > mmap(NULL, 2255216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c6574a000 > mprotect(0x7f8c6576e000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c6596d000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f8c6596d000 > mmap(0x7f8c6596f000, 6512, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6596f000 > close(3) = 0 > open("/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3 > read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 > \26\0\0\0\0\0\0"..., 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=20024, ...}) = 0 > mmap(NULL, 2114112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c65545000 > mprotect(0x7f8c65549000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c65748000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c65748000 > close(3) = 0 > open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\37\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=37056, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b82000 > mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c6533c000 > mprotect(0x7f8c65343000, 2097152, PROT_NONE) = 0 > mmap(0x7f8c65543000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f8c65543000 > close(3) = 0 > open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 > read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 > \34\2\0\0\0\0\0"..., 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=2107816, ...}) = 0 > mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c64f7b000 > mprotect(0x7f8c65131000, 2097152, PROT_NONE) = 0 > mmap(0x7f8c65331000, 24576, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7f8c65331000 > mmap(0x7f8c65337000, 16960, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c65337000 > close(3) = 0 > open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=398272, ...}) = 0 > mmap(NULL, 2490888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c64d1a000 > mprotect(0x7f8c64d7a000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c64f79000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5f000) = 0x7f8c64f79000 > close(3) = 0 > open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=19512, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b81000 > mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c64b16000 > mprotect(0x7f8c64b19000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c64d18000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f8c64d18000 > close(3) = 0 > open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=19888, ...}) = 0 > mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c64911000 > mprotect(0x7f8c64915000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c64b14000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c64b14000 > close(3) = 0 > open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 > read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=142296, ...}) = 0 > mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c646f5000 > mprotect(0x7f8c6470b000, 2097152, PROT_NONE) = 0 > mmap(0x7f8c6490b000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f8c6490b000 > mmap(0x7f8c6490d000, 13408, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6490d000 > close(3) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b80000 > mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b7e000 > arch_prctl(ARCH_SET_FS, 0x7f8c65b7e800) = 0 > mprotect(0x7f8c65331000, 16384, PROT_READ) = 0 > mprotect(0x7f8c6490b000, 4096, PROT_READ) = 0 > mprotect(0x7f8c64b14000, 4096, PROT_READ) = 0 > mprotect(0x7f8c64d18000, 4096, PROT_READ) = 0 > mprotect(0x7f8c64f79000, 4096, PROT_READ) = 0 > mprotect(0x7f8c65543000, 4096, PROT_READ) = 0 > mprotect(0x7f8c65748000, 4096, PROT_READ) = 0 > mprotect(0x7f8c6596d000, 4096, PROT_READ) = 0 > mprotect(0x61a000, 4096, PROT_READ) = 0 > mprotect(0x7f8c65b92000, 4096, PROT_READ) = 0 > munmap(0x7f8c65b83000, 53575) = 0 > set_tid_address(0x7f8c65b7ead0) = 121484 > set_robust_list(0x7f8c65b7eae0, 24) = 0 > rt_sigaction(SIGRTMIN, {0x7f8c646fb780, [], SA_RESTORER|SA_SIGINFO, > 0x7f8c64704100}, NULL, 8) = 0 > rt_sigaction(SIGRT_1, {0x7f8c646fb810, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, > 0x7f8c64704100}, NULL, 8) = 0 > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0 > statfs("/sys/fs/selinux", 0x7ffeb752e770) = -1 ENOENT (No such file or > directory) > statfs("/selinux", 0x7ffeb752e770) = -1 ENOENT (No such file or > directory) > brk(0) = 0x155f000 > brk(0x1580000) = 0x1580000 > open("/proc/filesystems", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b90000 > read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 1024) = 377 > stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", > 0x7ffeb752e650) = -1 ENOENT (No such file or directory) > read(3, "", 1024) = 0 > close(3) = 0 > munmap(0x7f8c65b90000, 4096) = 0 > access("/etc/selinux/config", F_OK) = 0 > open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=106065056, ...}) = 0 > mmap(NULL, 106065056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c5e1ce000 > close(3) = 0 > ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, > {B38400 opost isig icanon echo ...}) = 0 > ioctl(1, TIOCGWINSZ, {ws_row=68, ws_col=252, ws_xpixel=0, ws_ypixel=0}) = 0 > open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b90000 > read(3, "# Locale name alias data base.\n#"..., 4096) = 2502 > read(3, "", 4096) = 0 > close(3) = 0 > munmap(0x7f8c65b90000, 4096) = 0 > open("/usr/share/locale/en_US.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en_US.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en_US/LC_TIME/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT > (No such file or directory) > stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned", > 0x7ffeb752e2b0) = -1 ENOENT (No such file or directory) > open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=26254, ...}) = 0 > mmap(NULL, 26254, PROT_READ, MAP_SHARED, 3, 0) = 0x7f8c65b8a000 > close(3) = 0 > futex(0x7f8c65336a80, FUTEX_WAKE_PRIVATE, 2147483647) = 0 > lstat("base", {st_mode=S_IFDIR|0777, st_size=3929293109, ...}) = 0 > socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3 > connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = > -1 ENOENT (No such file or directory) > close(3) = 0 > socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3 > connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = > -1 ENOENT (No such file or directory) > close(3) = 0 > open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=1717, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b89000 > read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1717 > read(3, "", 4096) = 0 > close(3) = 0 > munmap(0x7f8c65b89000, 4096) = 0 > open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0 > mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b70000 > close(3) = 0 > open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3 > read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\"\0\0\0\0\0\0"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=57824, ...}) = 0 > mmap(NULL, 2168952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) > = 0x7f8c5dfbc000 > mprotect(0x7f8c5dfc7000, 2093056, PROT_NONE) = 0 > mmap(0x7f8c5e1c6000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f8c5e1c6000 > mmap(0x7f8c5e1c8000, 22648, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c5e1c8000 > close(3) = 0 > mprotect(0x7f8c5e1c6000, 4096, PROT_READ) = 0 > munmap(0x7f8c65b70000, 53575) = 0 > open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=1330, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b89000 > read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1330 > close(3) = 0 > munmap(0x7f8c65b89000, 4096) = 0 > socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3 > connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = > -1 ENOENT (No such file or directory) > close(3) = 0 > socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3 > connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = > -1 ENOENT (No such file or directory) > close(3) = 0 > open("/etc/group", O_RDONLY|O_CLOEXEC) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=597, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = > 0x7f8c65b89000 > read(3, "root:x:0:\nbin:x:1:\ndaemon:x:2:\ns"..., 4096) = 597 > close(3) = 0 > munmap(0x7f8c65b89000, 4096) = 0 > openat(AT_FDCWD, "base", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3 > getdents(3, /* 533 entries */, 32768) = 32720 > lstat("base/dbus-glib-0.100-7.el7.x86_64.rpm", {st_mode=S_IFREG|0777, > st_size=104756, ...}) = 0 > lstat("base/tomcat-admin-webapps-7.0.54-2.el7_1.noarch.rpm", > {st_mode=S_IFREG|0777, st_size=38872, ...}) = 0 > lstat("base/hunspell-gl-0.20080515-8.el7.noarch.rpm", > {st_mode=S_IFREG|0777, st_size=212276, ...}) = 0 > lstat("base/libreoffice-langpack-de-4.3.7.2-5.el7.x86_64.rpm", > {st_mode=S_IFREG|0777, st_size=7570712, ...}) = 0 > lstat("base/paktype-naskh-basic-fonts-4.1-3.el7.noarch.rpm", > {st_mode=S_IFREG|0777, st_size=436532, ...}) = 0 > lstat("base/openlmi-networking-0.3.0-3.el7.x86_64.rpm", > {st_mode=S_IFREG|0777, st_size=148344, ...}) = 0 > lstat("base/kdenetwork-kopete-libs-4.10.5-8.el7_0.x86_64.rpm", > {st_mode=S_IFREG|0777, st_size=1191908, ...}) = 0 > lstat("base/libwmf-0.2.8.4-41.el7_1.x86_64.rpm", {st_mode=S_IFREG|0777, > st_size=137716, ...}) = 0 > lstat("base/texlive-texconfig-bin-svn27344.0-38.20130427_r30134.el7.noarch.rpm", > {st_mode=S_IFREG|0777, st_size=16908, ...}) = 0 > lstat("base/gcc-c++-4.8.5-4.el7.x86_64.rpm", {st_mode=S_IFREG|0777, > st_size=7508064, ...}) = 0 > ========================================================== > > -- > Best Regards > Jian Zhang > > _______________________________________________ > ceph-users mailing list > [email protected] > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com >
_______________________________________________ ceph-users mailing list [email protected] http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
