Re: [gentoo-portage-dev] portage perfoemance again ...

2018-03-06 Thread Zac Medico
On 03/06/2018 02:10 PM, Joakim Tjernlund wrote:
> .. using portage 2.3.24 on our embedded target(ppc329 It takes forever, > 10 
> mins) to just do the
> dependencies for emerge -aNDuv world
> Here is what strace says, one can see portage is rereading the profile over 
> and over again.
> At this point I am prepared to try anything to eliminate the rereading of the 
> profile, any pointers?
>  
> 23:04:25.472818 _llseek(5, 0, [0], SEEK_CUR) = 0
> 23:04:25.472978 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 677
> 23:04:25.473099 read(5, "", 28672)  = 0
> 23:04:25.473228 read(5, "", 32768)  = 0
> 23:04:25.473366 close(5)= 0
> 23:04:25.482102 stat64("/usr/portage/profiles/base/profile.bashrc", 
> {st_mode=S_IFREG|0644, st_size=183, ...}) = 0
> 23:04:25.482509 stat64("/usr/portage/profiles/default/linux/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.482887 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.483240 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.483594 
> stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.483924 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.484269 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.484604 
> stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfe61330) 
> = -1 ENOENT (No such file or directory)
> 23:04:25.484930 
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.485272 stat64("/usr/portage/profiles/releases/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.485615 stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 
> 0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.485944 
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc",
>  0xbfe61330) = -1 ENOENT (No such file or directory)
> 23:04:25.486329 
> stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
>  {st_mode=S_IFREG|0644, st_size=133, ...}) = 0
> 23:04:25.486764 stat64("/etc/portage/profile/profile.bashrc", 0xbfe61330) = 
> -1 ENOENT (No such file or directory)

The profile.bashrc stat calls are repeated for every setcpv call here:

https://gitweb.gentoo.org/proj/portage.git/tree/pym/portage/package/ebuild/config.py?h=portage-2.3.24#n1614

We can easily perform the stat calls in the constructor, and save the
results in an attribute, like when we initialize self._pbashrcdict.

Bug filed: https://bugs.gentoo.org/649806
-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature


[gentoo-portage-dev] portage perfoemance again ...

2018-03-06 Thread Joakim Tjernlund
.. using portage 2.3.24 on our embedded target(ppc329 It takes forever, > 10 
mins) to just do the
dependencies for emerge -aNDuv world
Here is what strace says, one can see portage is rereading the profile over and 
over again.
At this point I am prepared to try anything to eliminate the rereading of the 
profile, any pointers?
 
23:04:25.472818 _llseek(5, 0, [0], SEEK_CUR) = 0
23:04:25.472978 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 677
23:04:25.473099 read(5, "", 28672)  = 0
23:04:25.473228 read(5, "", 32768)  = 0
23:04:25.473366 close(5)= 0
23:04:25.482102 stat64("/usr/portage/profiles/base/profile.bashrc", 
{st_mode=S_IFREG|0644, st_size=183, ...}) = 0
23:04:25.482509 stat64("/usr/portage/profiles/default/linux/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.482887 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.483240 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.483594 
stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.483924 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.484269 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.484604 
stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfe61330) = 
-1 ENOENT (No such file or directory)
23:04:25.484930 
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.485272 stat64("/usr/portage/profiles/releases/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.485615 stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.485944 
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.486329 
stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
 {st_mode=S_IFREG|0644, st_size=133, ...}) = 0
23:04:25.486764 stat64("/etc/portage/profile/profile.bashrc", 0xbfe61330) = -1 
ENOENT (No such file or directory)
23:04:25.499748 
access("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild",
 R_OK) = 0
23:04:25.500088 
stat64("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild",
 {st_mode=S_IFREG|0644, st_size=664, ...}) = 0
23:04:25.500417 
open("/usr/portage/metadata/md5-cache/virtual/perl-Text-ParseWords-3.290.0-r1", 
O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
23:04:25.500545 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0
23:04:25.500664 ioctl(5, TCGETS, 0xbfe61f98) = -1 ENOTTY (Inappropriate ioctl 
for device)
23:04:25.500761 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0
23:04:25.500897 _llseek(5, 0, [0], SEEK_CUR) = 0
23:04:25.501101 _llseek(5, 0, [0], SEEK_CUR) = 0
23:04:25.501238 _llseek(5, 0, [0], SEEK_CUR) = 0
23:04:25.501343 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0
23:04:25.501446 read(5, "DEFINED_PHASES=-\nDESCRIPTION=Vir"..., 594) = 593
23:04:25.501552 read(5, "", 1)  = 0
23:04:25.501831 fstat64(5, {st_mode=S_IFREG|0644, st_size=593, ...}) = 0
23:04:25.502013 close(5)= 0
23:04:25.502474 
open("/usr/portage/virtual/perl-Text-ParseWords/perl-Text-ParseWords-3.290.0-r1.ebuild",
 O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
23:04:25.502607 fstat64(5, {st_mode=S_IFREG|0644, st_size=664, ...}) = 0
23:04:25.502739 ioctl(5, TCGETS, 0xbfe61b88) = -1 ENOTTY (Inappropriate ioctl 
for device)
23:04:25.502838 fstat64(5, {st_mode=S_IFREG|0644, st_size=664, ...}) = 0
23:04:25.502961 _llseek(5, 0, [0], SEEK_CUR) = 0
23:04:25.503114 read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 664
23:04:25.503244 read(5, "", 28672)  = 0
23:04:25.503374 read(5, "", 32768)  = 0
23:04:25.503519 close(5)= 0
23:04:25.512617 stat64("/usr/portage/profiles/base/profile.bashrc", 
{st_mode=S_IFREG|0644, st_size=183, ...}) = 0
23:04:25.513003 stat64("/usr/portage/profiles/default/linux/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.513379 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.513736 stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.514142 
stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.514494 stat64("/usr/portage/profiles/arch/base/profile.bashrc", 
0xbfe61330) = -1 ENOENT (No such file or directory)
23:04:25.514831