Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod
You need to rerun with an "strace -f". That way it will also strace the
fork's one of which it appears to be waiting on.
On Mon, May 30, 2022 at 1:52 AM Abhishek Agarwal <
mragarwal.develo...@gmail.com> wrote:
> When a kubernetes pod is scheduled on the node having lvm2 libraries
> already installed and trying to run lvm commands using those node binaries
> from inside the pod container, the commands hang and are waiting on
> something to complete. Although when ctrl+c is pressed the terminal session
> resumes and checking the final code for the execution returns a "0" error
> code and the commands operation is also carried out successfully.
>
>
> Below is the command output and strace for the command:-
>
> strace of lvcreate on the pod container scheduled on the node where lvm2
> binaries are present:
> # strace /sbin/lvm-eg/lvcreate -n test-lv -L 1G shared-vg
> execve("/sbin/lvm-eg/lvcreate", ["/sbin/lvm-eg/lvcreate", "-n", "test-lv",
> "-L", "1G", "shared-vg"], 0x7ffe7d7a6c98 /* 37 vars */) = 0
> brk(NULL)= 0x55d196f41000
> arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdd8247a80) = -1 EINVAL (Invalid
> argument)
> access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
> directory)
> openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=8679, ...}) = 0
> mmap(NULL, 8679, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0d37895000
> close(3)= 0
> openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
> read(3,
> "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360A\2\0\0\0\0\0"..., 832)
> = 832
> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
> 784, 64) = 784
> pread64(3,
> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
> 848) = 32
> pread64(3,
> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
> 68, 880) = 68
> fstat(3, {st_mode=S_IFREG|0755, st_size=2029560, ...}) = 0
> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f0d37893000
> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
> 784, 64) = 784
> pread64(3,
> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
> 848) = 32
> pread64(3,
> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
> 68, 880) = 68
> mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
> 0x7f0d376a1000
> mmap(0x7f0d376c3000, 1540096, PROT_READ|PROT_EXEC,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f0d376c3000
> mmap(0x7f0d3783b000, 319488, PROT_READ,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f0d3783b000
> mmap(0x7f0d37889000, 24576, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f0d37889000
> mmap(0x7f0d3788f000, 13920, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0d3788f000
> close(3)= 0
> arch_prctl(ARCH_SET_FS, 0x7f0d37894580) = 0
> mprotect(0x7f0d37889000, 16384, PROT_READ) = 0
> mprotect(0x55d196662000, 8192, PROT_READ) = 0
> mprotect(0x7f0d378c5000, 4096, PROT_READ) = 0
> munmap(0x7f0d37895000, 8679) = 0
> getuid()= 0
> getgid()= 0
> getpid()= 19163
> rt_sigaction(SIGCHLD, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> geteuid()= 0
> brk(NULL)= 0x55d196f41000
> brk(0x55d196f62000) = 0x55d196f62000
> getppid()= 19160
> stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
> stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
> openat(AT_FDCWD, "/sbin/lvm-eg/lvcreate", O_RDONLY) = 3
> fcntl(3, F_DUPFD, 10) = 10
> close(3)= 0
> fcntl(10, F_SETFD, FD_CLOEXEC) = 0
> geteuid()= 0
> getegid()= 0
> rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGINT, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> read(10, "#!/bin/sh\npath=$(/sbin/lvm-eg/ge"..., 8192) = 79
> pipe([3, 4]) = 0
> clone(child_stack=NULL,
> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
> child_tidptr=0x7f0d37894850) = 19164
> close(4)= 0
> read(3, "/usr/sbin/lvcreate\n", 128) = 19
> --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19164, si_uid=0,
> si_status=0,
Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod
Dne 27. 05. 22 v 9:02 Abhishek Agarwal napsal(a): When a kubernetes pod is scheduled on the node having lvm2 libraries already installed and trying to run lvm commands using those node binaries from inside the pod container, the commands hang and are waiting on something to complete. Although when ctrl+c is pressed the terminal session resumes and checking the final code for the execution returns a "0" error code and the commands operation is also carried out successfully. lvm2 is *NOT* designed to be executed in/from a container. It cannot work properly as it directly communicates with system's udevd - which you likely don't have running in your container. You could kind of 'fake it' by running lvm2 wihout udev synchronization - but this will just open another cave of other problems (missing synchronization). So your lvm2 command should be always executed on your hosting machine (since it does control resources without containerization support - like devices) and then you should 'pass' created LV to your container in some way. Regards Zdenek ___ linux-lvm mailing list linux-lvm@redhat.com https://listman.redhat.com/mailman/listinfo/linux-lvm read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
Re: [linux-lvm] raid10 with missing redundancy, but health status claims it is ok.
On Mon, May 30, 2022 at 10:16:27AM +0200, Olaf Seibert wrote: > First, John, thanks for your reply. > > On 28.05.22 18:15, John Stoffel wrote: > >> "Olaf" == Olaf Seibert writes: > > > > I'm leaving for the rest of the weekend, but hopefully this will help you... > > > > Olaf> Hi all, I'm new to this list. I hope somebody here can help me. > > > > We will try! But I would strongly urge that you take backups of all > > your data NOW, before you do anything else. Copy to another disk > > which is seperate from this system just in case. > > Unfortunately there are some complicating factors that I left out so far. > The machine in question is a host for virtual machines run by customers. > So we can't just even look at the data, never mind rsyncing it. > (the name "nova" might have given that away; that is the name of the > OpenStack compute service) Can you try to live-migrate the VMs off of this node? If not, can you announce a maintenance window and power off the VMs so you can take a block-level backup? -- Sincerely, Demi Marie Obenour (she/her/hers) Invisible Things Lab signature.asc Description: PGP signature ___ linux-lvm mailing list linux-lvm@redhat.com https://listman.redhat.com/mailman/listinfo/linux-lvm read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
[linux-lvm] lvm commands hanging when run from inside a kubernetes pod
When a kubernetes pod is scheduled on the node having lvm2 libraries
already installed and trying to run lvm commands using those node binaries
from inside the pod container, the commands hang and are waiting on
something to complete. Although when ctrl+c is pressed the terminal session
resumes and checking the final code for the execution returns a "0" error
code and the commands operation is also carried out successfully.
Below is the command output and strace for the command:-
strace of lvcreate on the pod container scheduled on the node where lvm2
binaries are present:
# strace /sbin/lvm-eg/lvcreate -n test-lv -L 1G shared-vg
execve("/sbin/lvm-eg/lvcreate", ["/sbin/lvm-eg/lvcreate", "-n", "test-lv",
"-L", "1G", "shared-vg"], 0x7ffe7d7a6c98 /* 37 vars */) = 0
brk(NULL)= 0x55d196f41000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdd8247a80) = -1 EINVAL (Invalid
argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=8679, ...}) = 0
mmap(NULL, 8679, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0d37895000
close(3)= 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360A\2\0\0\0\0\0"..., 832)
= 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
784, 64) = 784
pread64(3,
"\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
848) = 32
pread64(3,
"\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029560, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f0d37893000
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
784, 64) = 784
pread64(3,
"\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
848) = 32
pread64(3,
"\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f0d376a1000
mmap(0x7f0d376c3000, 1540096, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f0d376c3000
mmap(0x7f0d3783b000, 319488, PROT_READ,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f0d3783b000
mmap(0x7f0d37889000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f0d37889000
mmap(0x7f0d3788f000, 13920, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0d3788f000
close(3)= 0
arch_prctl(ARCH_SET_FS, 0x7f0d37894580) = 0
mprotect(0x7f0d37889000, 16384, PROT_READ) = 0
mprotect(0x55d196662000, 8192, PROT_READ) = 0
mprotect(0x7f0d378c5000, 4096, PROT_READ) = 0
munmap(0x7f0d37895000, 8679) = 0
getuid()= 0
getgid()= 0
getpid()= 19163
rt_sigaction(SIGCHLD, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
geteuid()= 0
brk(NULL)= 0x55d196f41000
brk(0x55d196f62000) = 0x55d196f62000
getppid()= 19160
stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "/sbin/lvm-eg/lvcreate", O_RDONLY) = 3
fcntl(3, F_DUPFD, 10) = 10
close(3)= 0
fcntl(10, F_SETFD, FD_CLOEXEC) = 0
geteuid()= 0
getegid()= 0
rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8)
= 0
rt_sigaction(SIGINT, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
8) = 0
rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
8) = 0
rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
read(10, "#!/bin/sh\npath=$(/sbin/lvm-eg/ge"..., 8192) = 79
pipe([3, 4]) = 0
clone(child_stack=NULL,
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
child_tidptr=0x7f0d37894850) = 19164
close(4)= 0
read(3, "/usr/sbin/lvcreate\n", 128) = 19
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19164, si_uid=0,
si_status=0, si_utime=0, si_stime=0} ---
rt_sigreturn({mask=[]}) = 19
read(3, "", 128)= 0
close(3)= 0
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 19164
stat("/usr/local/sbin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file or
directory)
stat("/usr/local/bin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file or
directory)
stat("/usr/sbin/chroot", {st_mode=S_IFREG|0755,
