Re: [gentoo-portage-dev] portage perfoemance again ...
On Tue, 2018-03-06 at 15:11 -0800, Zac Medico wrote: > On 03/06/2018 02:10 PM, Joakim Tjernlund wrote: > > .. using portage 2.3.24 on our embedded target(ppc329 It takes forever, > > > 10 mins) to just do the > > dependencies for emerge -aNDuv world > > Here is what strace says, one can see portage is rereading the profile over > > and over again. > > At this point I am prepared to try anything to eliminate the rereading of > > the profile, any pointers? > > > > 23:04:25.472818 _llseek(5, 0, [0], SEEK_CUR) = 0 > > 23:04:25.472978 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 677 > > 23:04:25.473099 read(5, "", 28672) = 0 > > 23:04:25.473228 read(5, "", 32768) = 0 > > 23:04:25.473366 close(5)= 0 > > 23:04:25.482102 stat64("/usr/portage/profiles/base/profile.bashrc", > > {st_mode=S_IFREG|0644, st_size=183, ...}) = 0 > > 23:04:25.482509 > > stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfe61330) = > > -1 ENOENT (No such file or directory) > > 23:04:25.482887 stat64("/usr/portage/profiles/arch/base/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.483240 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.483594 > > stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.483924 stat64("/usr/portage/profiles/arch/base/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.484269 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.484604 > > stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.484930 > > stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.485272 stat64("/usr/portage/profiles/releases/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.485615 > > stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfe61330) = > > -1 ENOENT (No such file or directory) > > 23:04:25.485944 > > stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", > > 0xbfe61330) = -1 ENOENT (No such file or directory) > > 23:04:25.486329 > > stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc", > > {st_mode=S_IFREG|0644, st_size=133, ...}) = 0 > > 23:04:25.486764 stat64("/etc/portage/profile/profile.bashrc", 0xbfe61330) = > > -1 ENOENT (No such file or directory) > > The profile.bashrc stat calls are repeated for every setcpv call here: > > https://gitweb.gentoo.org/proj/portage.git/tree/pym/portage/package/ebuild/config.py?h=portage-2.3.24#n1614 > > We can easily perform the stat calls in the constructor, and save the > results in an attribute, like when we initialize self._pbashrcdict. > > Bug filed: https://bugs.gentoo.org/649806 Nice! One you got a patch I will try it out. Desperate, I downgraded to portage 2.2.26 and now portage runs a lot faster! No measurements though as I need to complete the upgrade of our system ASAP. Does this make any sense to you? Jocke
Re: [gentoo-portage-dev] portage perfoemance again ...
On 03/06/2018 02:10 PM, Joakim Tjernlund wrote: > .. using portage 2.3.24 on our embedded target(ppc329 It takes forever, > 10 > mins) to just do the > dependencies for emerge -aNDuv world > Here is what strace says, one can see portage is rereading the profile over > and over again. > At this point I am prepared to try anything to eliminate the rereading of the > profile, any pointers? > > 23:04:25.472818 _llseek(5, 0, [0], SEEK_CUR) = 0 > 23:04:25.472978 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 677 > 23:04:25.473099 read(5, "", 28672) = 0 > 23:04:25.473228 read(5, "", 32768) = 0 > 23:04:25.473366 close(5)= 0 > 23:04:25.482102 stat64("/usr/portage/profiles/base/profile.bashrc", > {st_mode=S_IFREG|0644, st_size=183, ...}) = 0 > 23:04:25.482509 stat64("/usr/portage/profiles/default/linux/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.482887 stat64("/usr/portage/profiles/arch/base/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.483240 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.483594 > stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.483924 stat64("/usr/portage/profiles/arch/base/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.484269 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.484604 > stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfe61330) > = -1 ENOENT (No such file or directory) > 23:04:25.484930 > stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.485272 stat64("/usr/portage/profiles/releases/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.485615 stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.485944 > stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", > 0xbfe61330) = -1 ENOENT (No such file or directory) > 23:04:25.486329 > stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc", > {st_mode=S_IFREG|0644, st_size=133, ...}) = 0 > 23:04:25.486764 stat64("/etc/portage/profile/profile.bashrc", 0xbfe61330) = > -1 ENOENT (No such file or directory) The profile.bashrc stat calls are repeated for every setcpv call here: https://gitweb.gentoo.org/proj/portage.git/tree/pym/portage/package/ebuild/config.py?h=portage-2.3.24#n1614 We can easily perform the stat calls in the constructor, and save the results in an attribute, like when we initialize self._pbashrcdict. Bug filed: https://bugs.gentoo.org/649806 -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] portage perfoemance again ...
.. using portage 2.3.24 on our embedded target(ppc329 It takes forever, > 10 mins) to just do the dependencies for emerge -aNDuv world Here is what strace says, one can see portage is rereading the profile over and over again. At this point I am prepared to try anything to eliminate the rereading of the profile, any pointers? 23:04:25.472818 _llseek(5, 0, [0], SEEK_CUR) = 0 23:04:25.472978 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 677 23:04:25.473099 read(5, "", 28672) = 0 23:04:25.473228 read(5, "", 32768) = 0 23:04:25.473366 close(5)= 0 23:04:25.482102 stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, st_size=183, ...}) = 0 23:04:25.482509 stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.482887 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.483240 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.483594 stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.483924 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.484269 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.484604 stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.484930 stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.485272 stat64("/usr/portage/profiles/releases/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.485615 stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.485944 stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.486329 stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc", {st_mode=S_IFREG|0644, st_size=133, ...}) = 0 23:04:25.486764 stat64("/etc/portage/profile/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.499748 access("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild", R_OK) = 0 23:04:25.500088 stat64("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild", {st_mode=S_IFREG|0644, st_size=664, ...}) = 0 23:04:25.500417 open("/usr/portage/metadata/md5-cache/virtual/perl-Text-ParseWords-3.290.0-r1", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5 23:04:25.500545 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0 23:04:25.500664 ioctl(5, TCGETS, 0xbfe61f98) = -1 ENOTTY (Inappropriate ioctl for device) 23:04:25.500761 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0 23:04:25.500897 _llseek(5, 0, [0], SEEK_CUR) = 0 23:04:25.501101 _llseek(5, 0, [0], SEEK_CUR) = 0 23:04:25.501238 _llseek(5, 0, [0], SEEK_CUR) = 0 23:04:25.501343 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0 23:04:25.501446 read(5, "DEFINED_PHASES=-\nDESCRIPTION=Vir"..., 594) = 593 23:04:25.501552 read(5, "", 1) = 0 23:04:25.501831 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0 23:04:25.502013 close(5)= 0 23:04:25.502474 open("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5 23:04:25.502607 fstat64(5, {st_mode=S_IFREG|0644, st_size=664, ...}) = 0 23:04:25.502739 ioctl(5, TCGETS, 0xbfe61b88) = -1 ENOTTY (Inappropriate ioctl for device) 23:04:25.502838 fstat64(5, {st_mode=S_IFREG|0644, st_size=664, ...}) = 0 23:04:25.502961 _llseek(5, 0, [0], SEEK_CUR) = 0 23:04:25.503114 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 664 23:04:25.503244 read(5, "", 28672) = 0 23:04:25.503374 read(5, "", 32768) = 0 23:04:25.503519 close(5)= 0 23:04:25.512617 stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, st_size=183, ...}) = 0 23:04:25.513003 stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.513379 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.513736 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.514142 stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.514494 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfe61330) = -1 ENOENT (No such file or directory) 23:04:25.514831 stat64("/usr/portage/profiles/arch/powerpc/profile