Re: [gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
On 08-01-2019 20:59:34 +, M. J. Everitt wrote: > On 08/01/19 19:15, Zac Medico wrote: > > On 1/8/19 5:42 AM, Fabian Groffen wrote: > >> The reporting of files remaining can look somewhat odd since the report > >> interval is hardcoded to be per 1000 objects. Adjust this interval to > >> be time based. This means that modern (fast) machines likely will never > >> see the countdown messages at all. On slow setups the message will be > >> informative that there is progress, albeit rather slowly. While at it, > >> report percentage done. > >> > >> Output before this patch: > >> > >> * checking 6158 files for package collisions > >> 5158 files remaining ... > >> 4158 files remaining ... > >> 3158 files remaining ... > >> 2158 files remaining ... > >> 1158 files remaining ... > >> 158 files remaining ... > >> > >> Possible output after this patch on a slower machine: > >> > >> * checking 6158 files for package collisions > >> 48% done, 3145 files remaining ... > >> 96% done, 192 files remaining ... > >> 100% done > >> > >> Signed-off-by: Fabian Groffen > >> --- > >> lib/portage/dbapi/vartree.py | 11 +-- > >> 1 file changed, 9 insertions(+), 2 deletions(-) > >> > >> diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py > >> index 4b91caea8..244195fad 100644 > >> --- a/lib/portage/dbapi/vartree.py > >> +++ b/lib/portage/dbapi/vartree.py > >> @@ -3475,13 +3475,18 @@ class dblink(object): > >>symlink_collisions = [] > >>destroot = self.settings['ROOT'] > >>totfiles = len(file_list) + len(symlink_list) > >> + tnow = time.time() > >> + tinterv = 2 # seconds > >> + ninterv = tnow + tinterv > >>showMessage(_(" %s checking %d files for package > >> collisions\n") % \ > >>(colorize("GOOD", "*"), totfiles)) > >>for i, (f, f_type) in enumerate(chain( > >>((f, "reg") for f in file_list), > >>((f, "sym") for f in symlink_list))): > >> - if i % 1000 == 0 and i != 0: > >> - showMessage(_("%d files remaining > >> ...\n") % (totfiles - i)) > >> + if time.time() > ninterv: > >> + showMessage(_("%3d%% done, %d files > >> remaining ...\n") % > >> + (i * 100 / totfiles, > >> totfiles - i)) > >> + ninterv = time.time() + tinterv > >> > >>dest_path = normalize_path( > >>os.path.join(destroot, > >> f.lstrip(os.path.sep))) > >> @@ -3570,6 +3575,8 @@ class dblink(object): > >>break > >>if stopmerge: > >>collisions.append(f) > >> + if tnow + tinterv < ninterv: > >> + showMessage(_("100% done\n")) > >>return collisions, dirs_ro, symlink_collisions, > >> plib_collisions > >> > >>def _lstat_inode_map(self, path_iter): > >> > > Please replace time.time() with portage.util.monotonic.monotonic(). Will do. > It's a bit of a nit-pick, granted, but can we ensure that the count-down's > remain padded/justified such that the numbers line up for easy at-a-glance > inspection ? The optimal standard looks somewhat like the pre-merge Sizes: > > * Final size of build directory: 2696 KiB (2.6 MiB) > * Final size of installed tree: 5372 KiB (5.2 MiB) > > Otherwise, I think this will be quite helpful. Thanks. I think I understand what you mean, let me see if I can do something without too much complexity. Thanks, Fabian -- Fabian Groffen Gentoo on a different level signature.asc Description: PGP signature
Re: [gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
On 08/01/19 19:15, Zac Medico wrote: > On 1/8/19 5:42 AM, Fabian Groffen wrote: >> The reporting of files remaining can look somewhat odd since the report >> interval is hardcoded to be per 1000 objects. Adjust this interval to >> be time based. This means that modern (fast) machines likely will never >> see the countdown messages at all. On slow setups the message will be >> informative that there is progress, albeit rather slowly. While at it, >> report percentage done. >> >> Output before this patch: >> >> * checking 6158 files for package collisions >> 5158 files remaining ... >> 4158 files remaining ... >> 3158 files remaining ... >> 2158 files remaining ... >> 1158 files remaining ... >> 158 files remaining ... >> >> Possible output after this patch on a slower machine: >> >> * checking 6158 files for package collisions >> 48% done, 3145 files remaining ... >> 96% done, 192 files remaining ... >> 100% done >> >> Signed-off-by: Fabian Groffen >> --- >> lib/portage/dbapi/vartree.py | 11 +-- >> 1 file changed, 9 insertions(+), 2 deletions(-) >> >> diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py >> index 4b91caea8..244195fad 100644 >> --- a/lib/portage/dbapi/vartree.py >> +++ b/lib/portage/dbapi/vartree.py >> @@ -3475,13 +3475,18 @@ class dblink(object): >> symlink_collisions = [] >> destroot = self.settings['ROOT'] >> totfiles = len(file_list) + len(symlink_list) >> +tnow = time.time() >> +tinterv = 2 # seconds >> +ninterv = tnow + tinterv >> showMessage(_(" %s checking %d files for package >> collisions\n") % \ >> (colorize("GOOD", "*"), totfiles)) >> for i, (f, f_type) in enumerate(chain( >> ((f, "reg") for f in file_list), >> ((f, "sym") for f in symlink_list))): >> -if i % 1000 == 0 and i != 0: >> -showMessage(_("%d files remaining >> ...\n") % (totfiles - i)) >> +if time.time() > ninterv: >> +showMessage(_("%3d%% done, %d files >> remaining ...\n") % >> +(i * 100 / totfiles, >> totfiles - i)) >> +ninterv = time.time() + tinterv >> >> dest_path = normalize_path( >> os.path.join(destroot, >> f.lstrip(os.path.sep))) >> @@ -3570,6 +3575,8 @@ class dblink(object): >> break >> if stopmerge: >> collisions.append(f) >> +if tnow + tinterv < ninterv: >> +showMessage(_("100% done\n")) >> return collisions, dirs_ro, symlink_collisions, >> plib_collisions >> >> def _lstat_inode_map(self, path_iter): >> > Please replace time.time() with portage.util.monotonic.monotonic(). It's a bit of a nit-pick, granted, but can we ensure that the count-down's remain padded/justified such that the numbers line up for easy at-a-glance inspection ? The optimal standard looks somewhat like the pre-merge Sizes: * Final size of build directory: 2696 KiB (2.6 MiB) * Final size of installed tree: 5372 KiB (5.2 MiB) Otherwise, I think this will be quite helpful. Thanks. signature.asc Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
On 1/8/19 5:42 AM, Fabian Groffen wrote: > The reporting of files remaining can look somewhat odd since the report > interval is hardcoded to be per 1000 objects. Adjust this interval to > be time based. This means that modern (fast) machines likely will never > see the countdown messages at all. On slow setups the message will be > informative that there is progress, albeit rather slowly. While at it, > report percentage done. > > Output before this patch: > > * checking 6158 files for package collisions > 5158 files remaining ... > 4158 files remaining ... > 3158 files remaining ... > 2158 files remaining ... > 1158 files remaining ... > 158 files remaining ... > > Possible output after this patch on a slower machine: > > * checking 6158 files for package collisions > 48% done, 3145 files remaining ... > 96% done, 192 files remaining ... > 100% done > > Signed-off-by: Fabian Groffen > --- > lib/portage/dbapi/vartree.py | 11 +-- > 1 file changed, 9 insertions(+), 2 deletions(-) > > diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py > index 4b91caea8..244195fad 100644 > --- a/lib/portage/dbapi/vartree.py > +++ b/lib/portage/dbapi/vartree.py > @@ -3475,13 +3475,18 @@ class dblink(object): > symlink_collisions = [] > destroot = self.settings['ROOT'] > totfiles = len(file_list) + len(symlink_list) > + tnow = time.time() > + tinterv = 2 # seconds > + ninterv = tnow + tinterv > showMessage(_(" %s checking %d files for package > collisions\n") % \ > (colorize("GOOD", "*"), totfiles)) > for i, (f, f_type) in enumerate(chain( > ((f, "reg") for f in file_list), > ((f, "sym") for f in symlink_list))): > - if i % 1000 == 0 and i != 0: > - showMessage(_("%d files remaining > ...\n") % (totfiles - i)) > + if time.time() > ninterv: > + showMessage(_("%3d%% done, %d files > remaining ...\n") % > + (i * 100 / totfiles, > totfiles - i)) > + ninterv = time.time() + tinterv > > dest_path = normalize_path( > os.path.join(destroot, > f.lstrip(os.path.sep))) > @@ -3570,6 +3575,8 @@ class dblink(object): > break > if stopmerge: > collisions.append(f) > + if tnow + tinterv < ninterv: > + showMessage(_("100% done\n")) > return collisions, dirs_ro, symlink_collisions, > plib_collisions > > def _lstat_inode_map(self, path_iter): > Please replace time.time() with portage.util.monotonic.monotonic(). -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
The reporting of files remaining can look somewhat odd since the report interval is hardcoded to be per 1000 objects. Adjust this interval to be time based. This means that modern (fast) machines likely will never see the countdown messages at all. On slow setups the message will be informative that there is progress, albeit rather slowly. While at it, report percentage done. Output before this patch: * checking 6158 files for package collisions 5158 files remaining ... 4158 files remaining ... 3158 files remaining ... 2158 files remaining ... 1158 files remaining ... 158 files remaining ... Possible output after this patch on a slower machine: * checking 6158 files for package collisions 48% done, 3145 files remaining ... 96% done, 192 files remaining ... 100% done Signed-off-by: Fabian Groffen --- lib/portage/dbapi/vartree.py | 11 +-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 4b91caea8..244195fad 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -3475,13 +3475,18 @@ class dblink(object): symlink_collisions = [] destroot = self.settings['ROOT'] totfiles = len(file_list) + len(symlink_list) + tnow = time.time() + tinterv = 2 # seconds + ninterv = tnow + tinterv showMessage(_(" %s checking %d files for package collisions\n") % \ (colorize("GOOD", "*"), totfiles)) for i, (f, f_type) in enumerate(chain( ((f, "reg") for f in file_list), ((f, "sym") for f in symlink_list))): - if i % 1000 == 0 and i != 0: - showMessage(_("%d files remaining ...\n") % (totfiles - i)) + if time.time() > ninterv: + showMessage(_("%3d%% done, %d files remaining ...\n") % + (i * 100 / totfiles, totfiles - i)) + ninterv = time.time() + tinterv dest_path = normalize_path( os.path.join(destroot, f.lstrip(os.path.sep))) @@ -3570,6 +3575,8 @@ class dblink(object): break if stopmerge: collisions.append(f) + if tnow + tinterv < ninterv: + showMessage(_("100% done\n")) return collisions, dirs_ro, symlink_collisions, plib_collisions def _lstat_inode_map(self, path_iter): -- 2.20.1