This happened again today, dead servers were not being removed. I had to stop heartbeat, and allow the resources to transfer to the second load balancer. Something is seriously wrong, but I don't know what it is. It doesn't seem to happen on the second load balancer.
Also, one ftp service vip always goes to the fallback on the primary lvs, but works correctly on the secondary (I hope this makes sense, it's early) Is there possibly a bug in the ftp monitoring of ldirectord? Michael -----Original Message----- From: Michael S. Moody [mailto:[EMAIL PROTECTED] Sent: Thursday, May 22, 2008 7:45 PM To: [email protected] Subject: Re: [lvs-users] Dead servers not being removed from pool, ldirectord Here's the output of strace against the pid for about 30 seconds: lvs1 fd # strace -p 10363 Process 10363 attached - interrupt to quit restart_syscall(<... resuming interrupted call ...>) = 0 close(3) = 0 open("/tmp/PerlIO_rsrzrG", O_RDWR|O_CREAT|O_EXCL, 0600) = 3 ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY (Inappropriate ioctl for device) lseek(3, 0, SEEK_CUR) = 0 unlink("/tmp/PerlIO_rsrzrG") = 0 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22 ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fcntl(22, F_GETFL) = 0x2 (flags O_RDWR) fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0 connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in progress) select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0}) connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = 0 fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK) fcntl(22, F_SETFL, O_RDWR) = 0 select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout) close(22) = 0 stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) = 0 rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0 rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 nanosleep({3, 0}, {3, 0}) = 0 close(3) = 0 open("/tmp/PerlIO_9tbzOK", O_RDWR|O_CREAT|O_EXCL, 0600) = 3 ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY (Inappropriate ioctl for device) lseek(3, 0, SEEK_CUR) = 0 unlink("/tmp/PerlIO_9tbzOK") = 0 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22 ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fcntl(22, F_GETFL) = 0x2 (flags O_RDWR) fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0 connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in progress) select(24, NULL, [22], NULL, {0, 0}) = 0 (Timeout) fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK) fcntl(22, F_SETFL, O_RDWR) = 0 close(22) = 0 stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) = 0 rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0 rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 nanosleep({3, 0}, {3, 0}) = 0 close(3) = 0 open("/tmp/PerlIO_IWMV4Y", O_RDWR|O_CREAT|O_EXCL, 0600) = 3 ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY (Inappropriate ioctl for device) lseek(3, 0, SEEK_CUR) = 0 unlink("/tmp/PerlIO_IWMV4Y") = 0 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22 ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fcntl(22, F_GETFL) = 0x2 (flags O_RDWR) fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0 connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in progress) select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0}) connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = 0 fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK) fcntl(22, F_SETFL, O_RDWR) = 0 select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout) close(22) = 0 stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) = 0 rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0 rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 nanosleep({3, 0}, {3, 0}) = 0 close(3) = 0 open("/tmp/PerlIO_ZQKIen", O_RDWR|O_CREAT|O_EXCL, 0600) = 3 ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY (Inappropriate ioctl for device) lseek(3, 0, SEEK_CUR) = 0 unlink("/tmp/PerlIO_ZQKIen") = 0 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22 ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fcntl(22, F_GETFL) = 0x2 (flags O_RDWR) fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0 connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in progress) select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0}) connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = 0 fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK) fcntl(22, F_SETFL, O_RDWR) = 0 select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout) close(22) = 0 stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) = 0 rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0 rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 nanosleep({3, 0}, {3, 0}) = 0 close(3) = 0 open("/tmp/PerlIO_FsEWhV", O_RDWR|O_CREAT|O_EXCL, 0600) = 3 ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY (Inappropriate ioctl for device) lseek(3, 0, SEEK_CUR) = 0 unlink("/tmp/PerlIO_FsEWhV") = 0 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0 fcntl(3, F_SETFD, FD_CLOEXEC) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 open("/etc/protocols", O_RDONLY) = 22 fcntl(22, F_GETFD) = 0 fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b31db397000 read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096 close(22) = 0 munmap(0x2b31db397000, 4096) = 0 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22 ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL (Invalid argument) lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) fcntl(22, F_SETFD, FD_CLOEXEC) = 0 fcntl(22, F_GETFL) = 0x2 (flags O_RDWR) fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0 connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in progress) select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0}) connect(22, {sa_family=AF_INET, sin_port=htons(21), sin_addr=inet_addr("192.168.1.195")}, 16) = 0 fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK) fcntl(22, F_SETFL, O_RDWR) = 0 select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout) close(22) = 0 stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) = 0 rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0 rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 nanosleep({3, 0}, ^C <unfinished ...> Process 10363 detached _______________________________________________ LinuxVirtualServer.org mailing list - [email protected] Send requests to [EMAIL PROTECTED] or go to http://lists.graemef.net/mailman/listinfo/lvs-users
