Hi,
When IPS GUI was released with 2008.05 we needed some things that IPS
did not provide, namely:
- An client API for consumers of IPS to code to.
- Fine grained progress notification, ProgressTracker did not provide
enough progress information for our needs.
- Ability to cancel running tasks.
So Michal lifted code from IPS itself and hacked on it. Not pretty but
it worked for the first rev. IPS GUI is currently broken (#2195, #2328)
as the underlying IPS has moved on from the 2008.05 release,
particularly with changes added for incorporations.
So what to do?
Well we need to:
- Introduce an API the IPS GUI and other clients including pkg(1) and
the installer can use.
- Update the ProgressTracker to allow a client to derive from it and get
whatever fine grained progress information they need.
- Add in support for canceling tasks. Currently IPS GUI uses a simple
threading model to allow canceling at the file level. A better solution
would be to allow canceling at any point in the download using an event
based networking model aka twisted, but we may have to stick with the
simple solution initially at least to get things back up and running for
IPS GUI.
To this end we are proposing the following attached changes for the
Progress (other changes for API and Canceling tasks to follow). Michal
and myself made them against the current IPS gate and they are not
tested yet. When they are we can create a Webrev, but as I'm off on
holidays for a week and a half I wanted to get these proposed changes
out for folks to take a look at.
The changes are broken down into mods for Evaluate, Download and Install
stages.
We could not see a way to get at file byte count easily in the current
code and if anyone has any idea how to do so we are all ears :) We'd
like to be able to give file byte count progress, not just he overall
byte count chunks currently available as progress. We'd also like to be
able to control the chunk size notification.
JR & Michal
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216118043 -3600
# Node ID cf2b4d30f7b1b5b4e6289f59b2fb86078a09a5a9
# Parent be3c47d9061ff2a855c1c12a39e7d02d105b5055
IPS GUI - Evaluate proposed changes
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216142807 -3600
# Node ID 56b4469abe423ec76df0f22d487ece5a3fb20858
# Parent cf2b4d30f7b1b5b4e6289f59b2fb86078a09a5a9
IPS GUI - Download proposed changes but without file byte count
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216203697 -3600
# Node ID f6692bf1b4050fa088d774c1258a99a77b9702a1
# Parent 56b4469abe423ec76df0f22d487ece5a3fb20858
IPS GUI - Install (Remove/Update/Install Phases) proposed changes
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216118043 -3600
# Node ID cf2b4d30f7b1b5b4e6289f59b2fb86078a09a5a9
# Parent be3c47d9061ff2a855c1c12a39e7d02d105b5055
IPS GUI - Evaluate proposed changes
diff -r be3c47d9061f -r cf2b4d30f7b1 src/modules/client/imageplan.py
--- a/src/modules/client/imageplan.py Fri Jul 11 18:24:48 2008 -0700
+++ b/src/modules/client/imageplan.py Tue Jul 15 11:34:03 2008 +0100
@@ -186,7 +186,7 @@
def evaluate_fmri(self, pfmri):
- self.progtrack.evaluate_progress()
+ self.progtrack.evaluate_progress(pfmri)
m = self.image.get_manifest(pfmri)
# [manifest] examine manifest for dependencies
@@ -273,6 +273,8 @@
def add_pkg_plan(self, pfmri):
"""add a pkg plan to imageplan for fully evaluated frmi"""
+
+ self.progtrack.evaluate_add_progress(pfmri)
m = self.image.get_manifest(pfmri)
pp = pkgplan.PkgPlan(self.image, self.progtrack)
@@ -288,6 +290,8 @@
def evaluate_fmri_removal(self, pfmri):
# prob. needs breaking up as well
+
+ self.progtrack.evaluate_rem_progress(pfmri)
assert self.image.has_manifest(pfmri)
dependents = self.image.get_dependents(pfmri)
@@ -345,7 +349,6 @@
# Operate on a copy, as it will be modified in flight.
for f in self.target_fmris[:]:
- self.progtrack.evaluate_progress()
try:
self.evaluate_fmri(f)
except KeyError, e:
@@ -359,11 +362,9 @@
for f in self.target_fmris:
self.add_pkg_plan(f)
- self.progtrack.evaluate_progress()
for f in self.target_rem_fmris[:]:
self.evaluate_fmri_removal(f)
- self.progtrack.evaluate_progress()
self.progtrack.evaluate_done()
diff -r be3c47d9061f -r cf2b4d30f7b1 src/modules/client/progress.py
--- a/src/modules/client/progress.py Fri Jul 11 18:24:48 2008 -0700
+++ b/src/modules/client/progress.py Tue Jul 15 11:34:03 2008 +0100
@@ -55,6 +55,10 @@
self.ver_cur_fmri = None
+ self.eval_cur_fmri = None
+ self.eval_add_cur_fmri = None
+ self.eval_rem_cur_fmri = None
+
self.dl_goal_nfiles = 0
self.dl_cur_nfiles = 0
self.dl_goal_nbytes = 0
@@ -81,8 +85,17 @@
def evaluate_start(self):
self.eval_output_start()
- def evaluate_progress(self):
+ def evaluate_progress(self, fmri):
+ self.eval_cur_fmri = fmri
self.eval_output_progress()
+
+ def evaluate_add_progress(self, fmri):
+ self.eval_add_cur_fmri = fmri
+ self.eval_output_add_progress()
+
+ def evaluate_rem_progress(self, fmri):
+ self.eval_rem_cur_fmri = fmri
+ self.eval_output_rem_progress()
def evaluate_done(self):
self.eval_output_done()
@@ -176,6 +189,12 @@
def eval_output_progress(self):
raise NotImplementedError("eval_output_progress() not implemented in superclass")
+ def eval_output_add_progress(self):
+ raise NotImplementedError("eval_output_add_progress() not implemented in superclass")
+
+ def eval_output_rem_progress(self):
+ raise NotImplementedError("eval_output_rem_progress() not implemented in superclass")
+
def eval_output_done(self):
raise NotImplementedError("eval_output_done() not implemented in superclass")
@@ -225,6 +244,10 @@
def eval_output_progress(self): return
+ def eval_output_add_progress(self): return
+
+ def eval_output_rem_progress(self): return
+
def eval_output_done(self): return
def ver_output(self): return
@@ -267,6 +290,10 @@
def eval_output_start(self): return
def eval_output_progress(self): return
+
+ def eval_output_add_progress(self): return
+
+ def eval_output_rem_progress(self): return
def eval_output_done(self): return
@@ -378,6 +405,12 @@
if e.errno == errno.EPIPE:
raise PipeError, e
raise
+
+ def eval_output_add_progress(self):
+ self.eval_output_progress()
+
+ def eval_output_rem_progress(self):
+ self.eval_output_progress()
def eval_output_progress(self):
if (time.time() - self.last_print_time) >= 0.10:
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216142807 -3600
# Node ID 56b4469abe423ec76df0f22d487ece5a3fb20858
# Parent cf2b4d30f7b1b5b4e6289f59b2fb86078a09a5a9
IPS GUI - Download proposed changes but without file byte count
diff -r cf2b4d30f7b1 -r 56b4469abe42 src/modules/client/filelist.py
--- a/src/modules/client/filelist.py Tue Jul 15 11:34:03 2008 +0100
+++ b/src/modules/client/filelist.py Tue Jul 15 18:26:47 2008 +0100
@@ -215,13 +215,15 @@
tarinfo.uname = "root"
tarinfo.gname = "root"
- # XXX catch IOError if tar stream closes inadvertently?
- tar_stream.extract_to(tarinfo, download_dir, hashval)
-
# Now that the file has been successfully extracted, move
# it to the cached content directory.
final_path = os.path.normpath(os.path.join(completed_dir,
hash_file_name(hashval)))
+
+ self.progtrack.download_start_file(final_path)
+
+ # XXX catch IOError if tar stream closes inadvertently?
+ tar_stream.extract_to(tarinfo, download_dir, hashval)
if not os.path.exists(os.path.dirname(final_path)):
os.makedirs(os.path.dirname(final_path))
@@ -244,6 +246,7 @@
# Remove successfully extracted items from the hash
# and adjust bean counters
self._del_hash(hashval)
+ self.progtrack.download_end_file()
def flush(self):
diff -r cf2b4d30f7b1 -r 56b4469abe42 src/modules/client/imageplan.py
--- a/src/modules/client/imageplan.py Tue Jul 15 11:34:03 2008 +0100
+++ b/src/modules/client/imageplan.py Tue Jul 15 18:26:47 2008 +0100
@@ -366,6 +366,24 @@
for f in self.target_rem_fmris[:]:
self.evaluate_fmri_removal(f)
+ npkgs = 0
+ nfiles = 0
+ nbytes = 0
+ nactions = 0
+ for p in self.pkg_plans:
+ nf, nb = p.get_xferstats()
+ nbytes += nb
+ nfiles += nf
+ nactions += p.get_nactions()
+
+ # It's not perfectly accurate but we count a download
+ # even if the package will do zero data transfer. This
+ # makes the pkg stats consistent between download and
+ # install.
+ npkgs += 1
+
+ # Require uptodate image goal information to be able to display to user before preexecute
+ self.progtrack.download_set_image_goal(npkgs, nfiles, nbytes)
self.progtrack.evaluate_done()
self.state = EVALUATED_OK
@@ -387,29 +405,11 @@
self.state = PREEXECUTED_OK
return
- npkgs = 0
- nfiles = 0
- nbytes = 0
- nactions = 0
try:
- for p in self.pkg_plans:
- nf, nb = p.get_xferstats()
- nbytes += nb
- nfiles += nf
- nactions += p.get_nactions()
-
- # It's not perfectly accurate but we count a download
- # even if the package will do zero data transfer. This
- # makes the pkg stats consistent between download and
- # install.
- npkgs += 1
-
- self.progtrack.download_set_goal(npkgs, nfiles, nbytes)
-
for p in self.pkg_plans:
p.preexecute()
- self.progtrack.download_done()
+ self.progtrack.download_image_done()
except:
self.state = PREEXECUTED_ERROR
raise
diff -r cf2b4d30f7b1 -r 56b4469abe42 src/modules/client/pkgplan.py
--- a/src/modules/client/pkgplan.py Tue Jul 15 11:34:03 2008 +0100
+++ b/src/modules/client/pkgplan.py Tue Jul 15 18:26:47 2008 +0100
@@ -235,8 +235,9 @@
at such a time.
"""
flist = filelist.FileList(self.image, self.destination_fmri,
- self.__progtrack)
+ self.__progtrack)
+ self.__progtrack.download_set_pkg_goal(self.get_xferstats())
self.__progtrack.download_start_pkg(self.get_xfername())
# retrieval step
@@ -261,7 +262,7 @@
# Tell flist to get any remaining files
flist.flush()
- self.__progtrack.download_end_pkg()
+ self.__progtrack.download_pkg_done()
def gen_install_actions(self):
for src, dest in self.actions[0]:
diff -r cf2b4d30f7b1 -r 56b4469abe42 src/modules/client/progress.py
--- a/src/modules/client/progress.py Tue Jul 15 11:34:03 2008 +0100
+++ b/src/modules/client/progress.py Tue Jul 15 18:26:47 2008 +0100
@@ -59,13 +59,24 @@
self.eval_add_cur_fmri = None
self.eval_rem_cur_fmri = None
- self.dl_goal_nfiles = 0
- self.dl_cur_nfiles = 0
- self.dl_goal_nbytes = 0
- self.dl_cur_nbytes = 0
- self.dl_goal_npkgs = 0
- self.dl_cur_npkgs = 0
- self.dl_cur_pkg = "None"
+ self.dl_goal_image_npkgs = 0
+ self.dl_goal_image_nfiles = 0
+ self.dl_goal_image_nbytes = 0
+
+ self.dl_cur_image_npkgs = 0
+ self.dl_cur_image_nfiles = 0
+ self.dl_cur_image_nbytes = 0
+
+ self.dl_goal_pkg_nfiles = 0
+ self.dl_goal_pkg_nbytes = 0
+
+ self.dl_cur_pkg_nfiles = 0
+ self.dl_cur_pkg_nbytes = 0
+ self.dl_cur_pkgname = "None"
+
+ self.dl_goal_file_nbytes = 0
+ self.dl_cur_file_nbytes = 0
+ self.dl_cur_filename = "None"
self.act_cur_nactions = 0
self.act_goal_nactions = 0
@@ -111,48 +122,85 @@
self.ver_cur_fmri = None
self.ver_output()
- def download_set_goal(self, npkgs, nfiles, nbytes):
- self.dl_goal_npkgs = npkgs
- self.dl_goal_nfiles = nfiles
- self.dl_goal_nbytes = nbytes
+ def download_set_image_goal(self, npkgs, nfiles, nbytes):
+ self.dl_goal_image_npkgs = npkgs
+ self.dl_goal_image_nfiles = nfiles
+ self.dl_goal_image_nbytes = nbytes
+
+ def download_set_pkg_goal(self, nfiles, nbytes):
+ self.dl_goal_pkg_nfiles= nfiles
+ self.dl_goal_pkg_nbytes = nbytes
def download_adjust_goal(self, npkgs, nfiles, nbytes):
- self.dl_goal_npkgs += npkgs
- self.dl_goal_nfiles += nfiles
- self.dl_goal_nbytes += nbytes
+ self.dl_goal_image_npkgs += npkgs
+ self.dl_goal_image_nfiles += nfiles
+ self.dl_goal_image_nbytes += nbytes
- assert self.dl_goal_npkgs >= 0
- assert self.dl_goal_nfiles >= 0
- assert self.dl_goal_nbytes >= 0
+ self.dl_goal_pkg_nfiles += nfiles
+ self.dl_goal_pkg_nbytes += nbytes
+
+ assert self.dl_goal_image_npkgs >= 0
+ assert self.dl_goal_image_nfiles >= 0
+ assert self.dl_goal_image_nbytes >= 0
+ assert self.dl_goal_pkg_nfiles >= 0
+ assert self.dl_goal_pkg_nbytes >= 0
def download_start_pkg(self, pkgname):
- self.dl_cur_pkg = pkgname
- if self.dl_goal_nbytes != 0:
+ self.dl_cur_pkgname = pkgname
+ self.dl_cur_pkg_nbytes = 0
+ self.dl_cur_pkg_nfiles = 0
+ if self.dl_goal_pkg_nbytes != 0:
self.dl_output()
def download_end_pkg(self):
self.dl_cur_npkgs += 1
- if self.dl_goal_nbytes != 0:
+ if self.dl_goal_pkg_nbytes != 0:
+ self.dl_output()
+
+ def download_start_file(self, filename):
+ self.dl_cur_filename = filename
+ self.dl_cur_file_nbytes = 0
+ if self.dl_goal_file_nbytes != 0:
+ self.dl_output()
+
+ def download_end_file(self):
+ self.dl_cur_nfiles += 1
+ if self.dl_goal_pkg_nbytes != 0:
+ self.dl_output()
+
+ def download_add_file_progress(self, nbytes):
+ """ Call to provide news that the file download has made progress """
+
+ self.dl_cur_file_nbytes += nbytes
+ if self.dl_goal_file_nbytes != 0:
self.dl_output()
def download_add_progress(self, nfiles, nbytes):
""" Call to provide news that the download has made progress """
- self.dl_cur_nbytes += nbytes
- self.dl_cur_nfiles += nfiles
- if self.dl_goal_nbytes != 0:
+ self.dl_cur_image_nbytes += nbytes
+ self.dl_cur_image_nfiles += nfiles
+ self.dl_cur_pkg_nbytes += nbytes
+ self.dl_cur_pkg_nfiles += nfiles
+ if self.dl_goal_image_nbytes != 0 and self.dl_goal_pkg_nbytes != 0:
self.dl_output()
- def download_done(self):
+ def download_image_done(self):
""" Call when all downloading is finished """
- if self.dl_goal_nbytes != 0:
+ if self.dl_goal_image_nbytes != 0:
self.dl_output_done()
- assert self.dl_cur_npkgs == self.dl_goal_npkgs
- assert self.dl_cur_nfiles == self.dl_goal_nfiles
- assert self.dl_cur_nbytes == self.dl_goal_nbytes
+ assert self.dl_cur_image_npkgs == self.dl_goal_image_npkgs
+ assert self.dl_cur_image_nfiles == self.dl_goal_image_nfiles
+ assert self.dl_cur_image_nbytes == self.dl_goal_image_nbytes
- def download_get_progress(self):
- return (self.dl_cur_npkgs, self.dl_cur_nfiles, self.dl_cur_nbytes)
+ def download_get_image_progress(self):
+ return (self.dl_cur_image_npkgs, self.dl_cur_image_nfiles, self.dl_cur_image_nbytes)
+
+ def download_get_pkg_progress(self):
+ return (self.dl_cur_pkg_nfiles, self.dl_cur_pkg_nbytes)
+
+ def download_get_file_progress(self):
+ return (self.dl_cur_file_nbytes)
def actions_set_goal(self, phase, nactions):
self.act_phase = phase
@@ -304,11 +352,11 @@
def dl_output(self):
try:
# The first time, emit header.
- if self.dl_cur_pkg != self.dl_last_printed_pkg:
+ if self.dl_cur_pkgname != self.dl_last_printed_pkg:
if self.dl_last_printed_pkg != None:
print "Done"
- print "Download: %s ... " % (self.dl_cur_pkg),
- self.dl_last_printed_pkg = self.dl_cur_pkg
+ print "Download: %s ... " % (self.dl_cur_pkgname),
+ self.dl_last_printed_pkg = self.dl_cur_pkgname
sys.stdout.flush()
except IOError, e:
if e.errno == errno.EPIPE:
@@ -491,12 +539,12 @@
else:
print self.cr,
print "%-40s %7s %11s %13s" % \
- (self.dl_cur_pkg,
- "%d/%d" % (self.dl_cur_npkgs, self.dl_goal_npkgs),
- "%d/%d" % (self.dl_cur_nfiles, self.dl_goal_nfiles),
+ (self.dl_cur_pkgname,
+ "%d/%d" % (self.dl_cur_image_npkgs, self.dl_goal_image_npkgs),
+ "%d/%d" % (self.dl_cur_image_nfiles, self.dl_goal_image_nfiles),
"%.2f/%.2f" % \
- ((self.dl_cur_nbytes / 1024.0 / 1024.0),
- (self.dl_goal_nbytes / 1024.0 / 1024.0))),
+ ((self.dl_cur_image_nbytes / 1024.0 / 1024.0),
+ (self.dl_goal_image_nbytes / 1024.0 / 1024.0))),
sys.stdout.flush()
except IOError, e:
if e.errno == errno.EPIPE:
@@ -504,7 +552,7 @@
raise
def dl_output_done(self):
- self.dl_cur_pkg = "Completed"
+ self.dl_cur_pkgname = "Completed"
self.dl_output()
try:
print
# HG changeset patch
# User [EMAIL PROTECTED]
# Date 1216203697 -3600
# Node ID f6692bf1b4050fa088d774c1258a99a77b9702a1
# Parent 56b4469abe423ec76df0f22d487ece5a3fb20858
IPS GUI - Install (Remove/Update/Install Phases) proposed changes
diff -r 56b4469abe42 -r f6692bf1b405 src/modules/client/imageplan.py
--- a/src/modules/client/imageplan.py Tue Jul 15 18:26:47 2008 +0100
+++ b/src/modules/client/imageplan.py Wed Jul 16 11:21:37 2008 +0100
@@ -384,6 +384,7 @@
# Require uptodate image goal information to be able to display to user before preexecute
self.progtrack.download_set_image_goal(npkgs, nfiles, nbytes)
+ self.progtrack.actions_set_total_goal(nactions)
self.progtrack.evaluate_done()
self.state = EVALUATED_OK
@@ -452,7 +453,7 @@
self.progtrack.actions_set_goal("Removal Phase", len(actions))
for p, src, dest in actions:
p.execute_removal(src, dest)
- self.progtrack.actions_add_progress()
+ self.progtrack.actions_add_progress(p.orign_fmri.get_fmri())
self.progtrack.actions_done()
# generate list of update actions, sort and execute
@@ -483,7 +484,8 @@
for p, src, dest in update_actions:
p.execute_update(src, dest)
- self.progtrack.actions_add_progress()
+ # XXX Destination should used here, check it is OK
+ self.progtrack.actions_add_progress(p.destination_fmri.get_fmri())
self.progtrack.actions_done()
@@ -495,8 +497,9 @@
for p, src, dest in install_actions:
p.execute_install(src, dest)
- self.progtrack.actions_add_progress()
+ self.progtrack.actions_add_progress(p.destination_fmri.get_fmri())
self.progtrack.actions_done()
+ self.progtrack.actions_total_done()
# handle any postexecute operations
diff -r 56b4469abe42 -r f6692bf1b405 src/modules/client/progress.py
--- a/src/modules/client/progress.py Tue Jul 15 18:26:47 2008 +0100
+++ b/src/modules/client/progress.py Wed Jul 16 11:21:37 2008 +0100
@@ -78,6 +78,8 @@
self.dl_cur_file_nbytes = 0
self.dl_cur_filename = "None"
+ self.act_cur_total_nactions = 0
+ self.act_goal_total_nactions = 0
self.act_cur_nactions = 0
self.act_goal_nactions = 0
self.act_phase = "None"
@@ -202,20 +204,30 @@
def download_get_file_progress(self):
return (self.dl_cur_file_nbytes)
+ def actions_set_total_goal(self, nactions):
+ self.act_goal_total_nactions = nactions
+ self.act_cur_total_nactions = 0
+
def actions_set_goal(self, phase, nactions):
self.act_phase = phase
self.act_goal_nactions = nactions
self.act_cur_nactions = 0
- def actions_add_progress(self):
+ def actions_add_progress(self, fmri = None):
self.act_cur_nactions += 1
+ self.act_cur_total_nactions += 1
if self.act_goal_nactions > 0:
- self.act_output()
+ self.act_output(fmri)
def actions_done(self):
if self.act_goal_nactions > 0:
self.act_output_done()
assert self.act_goal_nactions == self.act_cur_nactions
+
+ def actions_total_done(self):
+ if self.act_goal_total_nactions > 0:
+ self.act_output_total_done()
+ assert self.act_goal_total_nactions == self.act_cur_total_nactions
def actions_get_progress(self):
msg("not yet")
@@ -258,12 +270,14 @@
def dl_output_done(self):
raise NotImplementedError("dl_output_done() not implemented in superclass")
- def act_output(self):
+ def act_output(self, fmri):
raise NotImplementedError("act_output() not implemented in superclass")
def act_output_done(self):
raise NotImplementedError("act_output_done() not implemented in superclass")
+ def act_output_total_done(self):
+ raise NotImplementedError("act_output_done() not implemented in superclass")
class ProgressTrackerException(Exception):
@@ -306,7 +320,7 @@
def dl_output_done(self): return
- def act_output(self): return
+ def act_output(self, fmri): return
def act_output_done(self): return
@@ -372,7 +386,7 @@
raise PipeError, e
raise
- def act_output(self):
+ def act_output(self, fmri):
if self.act_phase != self.act_phase_last:
try:
print "%s ... " % self.act_phase,
@@ -563,7 +577,7 @@
raise PipeError, e
raise
- def act_output(self, force = False):
+ def act_output(self, fmri, force = False):
if force or (time.time() - self.last_print_time) >= 0.05:
self.last_print_time = time.time()
else:
@@ -591,7 +605,7 @@
raise
def act_output_done(self):
- self.act_output(force=True)
+ self.act_output(fmri = None, force=True)
try:
print
sys.stdout.flush()
_______________________________________________
pkg-discuss mailing list
[email protected]
http://mail.opensolaris.org/mailman/listinfo/pkg-discuss
