On Wed, Jul 5, 2017 at 9:46 AM, Nadav Goldin <ngol...@redhat.com> wrote:
> Hi, > Did it happen more than once? > No > > Looking at the logs what happened was: > 1. Lago checked the engine was SSH reachable - this was true. > 2. Then it tried connecting via SSH and collect the logs and timed out. > > On (1) we have retries and guards, on (2) we don't, as we assume (1) > just passed. I guess in some conditions that logic can be flawed. Can > you open an issue[1]? > Will do > > I'll try to fix it as soon as possible. > > Thanks, > > > [1] https://github.com/lago-project/lago/issues > Nadav. > > On Wed, Jul 5, 2017 at 9:07 AM, Gil Shinar <gshi...@redhat.com> wrote: > > Hi Nadav/Gal, > > > > I see the folowing exceptions in lago log: > > > > 2017-07-04 14:24:39,254::log_utils.py::__exit__::606::lago.prefix:: > DEBUG:: > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in > > _collect_artifacts > > vm.collect_artifacts(path, ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, > in > > collect_artifacts > > ignore_nopath=ignore_nopath > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, > in > > extract_paths > > return self.provider.extract_paths(paths, *args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", > line > > 297, in extract_paths > > ignore_nopath=ignore_nopath, > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, > in > > extract_paths > > self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, > in > > _extract_paths_scp > > propagate_fail=False > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, > in > > copy_from > > local_path=local_path, > > File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get > > self._recv_all() > > File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all > > msg = self.channel.recv(1024) > > File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line > 615, in > > recv > > raise socket.timeout() > > > > 2017-07-04 > > 14:24:39,254::utils.py::_ret_via_queue::60::lago.utils::ERROR::Error > while > > running thread > > Traceback (most recent call last): > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in > > _ret_via_queue > > queue.put({'return': func()}) > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in > > _collect_artifacts > > vm.collect_artifacts(path, ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, > in > > collect_artifacts > > ignore_nopath=ignore_nopath > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, > in > > extract_paths > > return self.provider.extract_paths(paths, *args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", > line > > 297, in extract_paths > > ignore_nopath=ignore_nopath, > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, > in > > extract_paths > > self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, > in > > _extract_paths_scp > > propagate_fail=False > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, > in > > copy_from > > local_path=local_path, > > File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get > > self._recv_all() > > File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all > > msg = self.channel.recv(1024) > > File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line > 615, in > > recv > > raise socket.timeout() > > timeout > > 2017-07-04 14:24:39,255::log_utils.py::end_log_task::669::root::ERROR::@ > > Collect artifacts: [31mERROR [0m (in 0:00:05) > > 2017-07-04 14:24:39,256::log_utils.py::__exit__::606::lago.prefix:: > DEBUG:: > > File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635, in > > wrapper > > return func(*args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in > > collect_artifacts > > self.virt_env.get_vms().values(), > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in > > invoke_in_parallel > > return vt.join_all() > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in > > _ret_via_queue > > queue.put({'return': func()}) > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in > > _collect_artifacts > > vm.collect_artifacts(path, ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, > in > > collect_artifacts > > ignore_nopath=ignore_nopath > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, > in > > extract_paths > > return self.provider.extract_paths(paths, *args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", > line > > 297, in extract_paths > > ignore_nopath=ignore_nopath, > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, > in > > extract_paths > > self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, > in > > _extract_paths_scp > > propagate_fail=False > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, > in > > copy_from > > local_path=local_path, > > File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get > > self._recv_all() > > File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all > > msg = self.channel.recv(1024) > > File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line > 615, in > > recv > > raise socket.timeout() > > > > 2017-07-04 14:24:39,256::cmd.py::main::960::cli::ERROR::Error occured, > > aborting > > Traceback (most recent call last): > > File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 954, in main > > cli_plugins[args.verb].do_run(args) > > File "/usr/lib/python2.7/site-packages/lago/plugins/cli.py", line > 184, in > > do_run > > self._do_run(**vars(args)) > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 501, in > > wrapper > > return func(*args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 512, in > > wrapper > > return func(*args, prefix=prefix, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 751, in > > do_collect > > prefix.collect_artifacts(output, ignore_nopath=not no_skip) > > File "/usr/lib/python2.7/site-packages/lago/sdk_utils.py", line 50, in > > wrapped > > return func(*args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635, > in > > wrapper > > return func(*args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in > > collect_artifacts > > self.virt_env.get_vms().values(), > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in > > invoke_in_parallel > > return vt.join_all() > > File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in > > _ret_via_queue > > queue.put({'return': func()}) > > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in > > _collect_artifacts > > vm.collect_artifacts(path, ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, > in > > collect_artifacts > > ignore_nopath=ignore_nopath > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, > in > > extract_paths > > return self.provider.extract_paths(paths, *args, **kwargs) > > File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", > line > > 297, in extract_paths > > ignore_nopath=ignore_nopath, > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, > in > > extract_paths > > self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath) > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, > in > > _extract_paths_scp > > propagate_fail=False > > File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, > in > > copy_from > > local_path=local_path, > > File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get > > self._recv_all() > > File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all > > msg = self.channel.recv(1024) > > File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line > 615, in > > recv > > raise socket.timeout() > > timeout > > > > > > Link to build: > > http://jenkins.ovirt.org/job/test-repo_ovirt_experimental_master/7459 > > > > Thanks > > Gil >
_______________________________________________ Infra mailing list Infra@ovirt.org http://lists.ovirt.org/mailman/listinfo/infra