Hello Calin,
I might be wrong, but this:
path_to_sstate_obj='/home/calin',
does not seem right.
Regards,
Cristian
-----Original Message-----
From: [email protected] [mailto:[email protected]]
On Behalf Of Calin Dragomir
Sent: Wednesday, June 12, 2013 12:30 PM
To: [email protected]
Subject: [Webhob] [PATCH 3/3] Create Data Store Interface (DSI) file
This adds the first version of the DSI file. It uses the Knotty code for now,
but inserts task related information into the database using the Django ORM
---
bitbake/lib/bb/ui/dsi.py | 609 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 609 insertions(+)
create mode 100644 bitbake/lib/bb/ui/dsi.py
diff --git a/bitbake/lib/bb/ui/dsi.py b/bitbake/lib/bb/ui/dsi.py new file mode
100644 index 0000000..4bcd509
--- /dev/null
+++ b/bitbake/lib/bb/ui/dsi.py
@@ -0,0 +1,609 @@
+#
+# BitBake (No)TTY UI Implementation
+#
+# Handling output to TTYs or files (no TTY) # # Copyright (C) 2006-2012
+Richard Purdie # # This program is free software; you can redistribute
+it and/or modify # it under the terms of the GNU General Public License
+version 2 as # published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, # but
+WITHOUT ANY WARRANTY; without even the implied warranty of #
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU
+General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+along # with this program; if not, write to the Free Software
+Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from __future__ import division
+
+import os
+import sys
+import xmlrpclib
+import logging
+import progressbar
+import signal
+import bb.msg
+import time
+import fcntl
+import struct
+import copy
+import datetime
+
+from bb.ui import uihelper
+
+logger = logging.getLogger("BitBake")
+interactive = sys.stdout.isatty()
+
+class WebHOBHelper(object):
+
+ def __init__(self):
+ self.configure_django()
+ self.task_order = 0
+ self.tasks_information = {}
+
+ def configure_django(self):
+ import webhob.whbmain.settings as whb_django_settings
+ from django.core.management import setup_environ
+ setup_environ(whb_django_settings)
+ # Add webhob to sys path for importing modules
+
+ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.d
+ irname(__file__))), 'webhob'))
+
+ def store_started_task(self, event, uuid):
+ self.task_order += 1
+ self.tasks_information[event.pid] = {
+ 'uuid': uuid,
+ 'task_id': 1,
+ 'task_executed': True,
+ 'order': self.task_order,
+ 'recipe': event._package,
+ 'task_name': event._task,
+ 'start_time': time.time(),
+ }
+
+ def update_stored_tasks(self, event, uuid):
+ self.tasks_information[event.pid].update({
+ 'outcome': event.getDisplayName().lower(),
+ 'end_time': time.time(),
+ })
+
+ def write_in_database(self, tasks_information):
+ # This needs to be imported after we have configured the Django
settings file
+ from webhob.orm.models import Tasks
+ for pid in tasks_information.keys():
+ task = tasks_information[pid]
+
+ task_obj = Tasks.objects.create(uuid=task['uuid'],
+ order=task['order'],
+ outcome=task['outcome'],
+ recipe=task['recipe'],
+ task_name=task['task_name'],
+ elapsed_time=task['end_time'] -
task['start_time'],
+ # The next lines are just for
testing reasons
+ task_id=1,
+ path_to_sstate_obj='/home/calin',
+ source_url='/',
+ log_file='/',
+ work_directory='/',
+ script_type='python',
+ file_path='/',
+ line_number=55,
+ py_stack_trace='Testing traceback',
+ disk_io = 23.55633,
+ cpu_usage=43,
+ dependent_tasks='Task 1',
+ errors_no=0,
+ warnings_no=0,
+ error='None',
+ warning='None',
+ sstate_result='not_applicable')
+ task_obj.save()
+
+class BBProgress(progressbar.ProgressBar):
+ def __init__(self, msg, maxval):
+ self.msg = msg
+ widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
+ progressbar.ETA()]
+
+ try:
+ self._resize_default = signal.getsignal(signal.SIGWINCH)
+ except:
+ self._resize_default = None
+ progressbar.ProgressBar.__init__(self, maxval, [self.msg + ":
+ "] + widgets)
+
+ def _handle_resize(self, signum, frame):
+ progressbar.ProgressBar._handle_resize(self, signum, frame)
+ if self._resize_default:
+ self._resize_default(signum, frame)
+ def finish(self):
+ progressbar.ProgressBar.finish(self)
+ if self._resize_default:
+ signal.signal(signal.SIGWINCH, self._resize_default)
+
+class NonInteractiveProgress(object):
+ fobj = sys.stdout
+
+ def __init__(self, msg, maxval):
+ self.msg = msg
+ self.maxval = maxval
+
+ def start(self):
+ self.fobj.write("%s..." % self.msg)
+ self.fobj.flush()
+ return self
+
+ def update(self, value):
+ pass
+
+ def finish(self):
+ self.fobj.write("done.\n")
+ self.fobj.flush()
+
+def new_progress(msg, maxval):
+ if interactive:
+ return BBProgress(msg, maxval)
+ else:
+ return NonInteractiveProgress(msg, maxval)
+
+def pluralise(singular, plural, qty):
+ if(qty == 1):
+ return singular % qty
+ else:
+ return plural % qty
+
+
+class InteractConsoleLogFilter(logging.Filter):
+ def __init__(self, tf, format):
+ self.tf = tf
+ self.format = format
+
+ def filter(self, record):
+ if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or
record.msg.startswith("recipe ")):
+ return False
+ self.tf.clearFooter()
+ return True
+
+class TerminalFilter(object):
+ columns = 80
+
+ def sigwinch_handle(self, signum, frame):
+ self.columns = self.getTerminalColumns()
+ if self._sigwinch_default:
+ self._sigwinch_default(signum, frame)
+
+ def getTerminalColumns(self):
+ def ioctl_GWINSZ(fd):
+ try:
+ cr = struct.unpack('hh', fcntl.ioctl(fd,
self.termios.TIOCGWINSZ, '1234'))
+ except:
+ return None
+ return cr
+ cr = ioctl_GWINSZ(sys.stdout.fileno())
+ if not cr:
+ try:
+ fd = os.open(os.ctermid(), os.O_RDONLY)
+ cr = ioctl_GWINSZ(fd)
+ os.close(fd)
+ except:
+ pass
+ if not cr:
+ try:
+ cr = (env['LINES'], env['COLUMNS'])
+ except:
+ cr = (25, 80)
+ return cr[1]
+
+ def __init__(self, main, helper, console, format):
+ self.main = main
+ self.helper = helper
+ self.cuu = None
+ self.stdinbackup = None
+ self.interactive = sys.stdout.isatty()
+ self.footer_present = False
+ self.lastpids = []
+
+ if not self.interactive:
+ return
+
+ try:
+ import curses
+ except ImportError:
+ sys.exit("FATAL: The knotty ui could not load the required
+ curses python module.")
+
+ import termios
+ self.curses = curses
+ self.termios = termios
+ try:
+ fd = sys.stdin.fileno()
+ self.stdinbackup = termios.tcgetattr(fd)
+ new = copy.deepcopy(self.stdinbackup)
+ new[3] = new[3] & ~termios.ECHO
+ termios.tcsetattr(fd, termios.TCSADRAIN, new)
+ curses.setupterm()
+ if curses.tigetnum("colors") > 2:
+ format.enable_color()
+ self.ed = curses.tigetstr("ed")
+ if self.ed:
+ self.cuu = curses.tigetstr("cuu")
+ try:
+ self._sigwinch_default = signal.getsignal(signal.SIGWINCH)
+ signal.signal(signal.SIGWINCH, self.sigwinch_handle)
+ except:
+ pass
+ self.columns = self.getTerminalColumns()
+ except:
+ self.cuu = None
+ console.addFilter(InteractConsoleLogFilter(self, format))
+
+ def clearFooter(self):
+ if self.footer_present:
+ lines = self.footer_present
+ sys.stdout.write(self.curses.tparm(self.cuu, lines))
+ sys.stdout.write(self.curses.tparm(self.ed))
+ self.footer_present = False
+
+ def updateFooter(self):
+ if not self.cuu:
+ return
+ activetasks = self.helper.running_tasks
+ failedtasks = self.helper.failed_tasks
+ runningpids = self.helper.running_pids
+ if self.footer_present and (self.lastcount ==
self.helper.tasknumber_current) and (self.lastpids == runningpids):
+ return
+ if self.footer_present:
+ self.clearFooter()
+ if (not self.helper.tasknumber_total or self.helper.tasknumber_current
== self.helper.tasknumber_total) and not len(activetasks):
+ return
+ tasks = []
+ for t in runningpids:
+ tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+
+ if self.main.shutdown:
+ content = "Waiting for %s running tasks to finish:" %
len(activetasks)
+ elif not len(activetasks):
+ content = "No currently running tasks (%s of %s)" %
(self.helper.tasknumber_current, self.helper.tasknumber_total)
+ else:
+ content = "Currently %s running tasks (%s of %s):" %
(len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+ print(content)
+ lines = 1 + int(len(content) / (self.columns + 1))
+ for tasknum, task in enumerate(tasks):
+ content = "%s: %s" % (tasknum, task)
+ print(content)
+ lines = lines + 1 + int(len(content) / (self.columns + 1))
+ self.footer_present = lines
+ self.lastpids = runningpids[:]
+ self.lastcount = self.helper.tasknumber_current
+
+ def finish(self):
+ if self.stdinbackup:
+ fd = sys.stdin.fileno()
+ self.termios.tcsetattr(fd, self.termios.TCSADRAIN,
+ self.stdinbackup)
+
+def main(server, eventHandler, params, tf = TerminalFilter):
+ print "This is running in DSI mode"
+
+ # Generate an unique ID for this build
+ # TODO: for multiple build commands this might not work as expected
+ import uuid
+ uuid = sessionid = str(uuid.uuid4())
+
+ # Get values of variables which control our output
+ includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
+ if error:
+ logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" %
error)
+ return 1
+ loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
+ if error:
+ logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable:
%s" % error)
+ return 1
+ consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"])
+ if error:
+ logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" %
error)
+ return 1
+
+ if sys.stdin.isatty() and sys.stdout.isatty():
+ log_exec_tty = True
+ else:
+ log_exec_tty = False
+
+ helper = uihelper.BBUIHelper()
+
+ console = logging.StreamHandler(sys.stdout)
+ format_str = "%(levelname)s: %(message)s"
+ format = bb.msg.BBLogFormatter(format_str)
+ bb.msg.addDefaultlogFilter(console)
+ console.setFormatter(format)
+ logger.addHandler(console)
+
+ if consolelogfile and not params.options.show_environment:
+ bb.utils.mkdirhier(os.path.dirname(consolelogfile))
+ conlogformat = bb.msg.BBLogFormatter(format_str)
+ consolelog = logging.FileHandler(consolelogfile)
+ bb.msg.addDefaultlogFilter(consolelog)
+ consolelog.setFormatter(conlogformat)
+ logger.addHandler(consolelog)
+
+ try:
+ params.updateFromServer(server)
+ cmdline = params.parseActions()
+ if not cmdline:
+ print("Nothing to do. Use 'bitbake world' to build everything, or run
'bitbake --help' for usage information.")
+ return 1
+ if 'msg' in cmdline and cmdline['msg']:
+ logger.error(cmdline['msg'])
+ return 1
+
+ ret, error = server.runCommand(cmdline['action'])
+ if error:
+ logger.error("Command '%s' failed: %s" % (cmdline, error))
+ return 1
+ elif ret != True:
+ logger.error("Command '%s' failed: returned %s" % (cmdline, ret))
+ return 1
+ except xmlrpclib.Fault as x:
+ logger.error("XMLRPC Fault getting commandline:\n %s" % x)
+ return 1
+
+ parseprogress = None
+ cacheprogress = None
+ main.shutdown = 0
+ interrupted = False
+ return_value = 0
+ errors = 0
+ warnings = 0
+ taskfailures = []
+ wbhbhelper = WebHOBHelper()
+
+ termfilter = tf(main, helper, console, format)
+
+ while True:
+ try:
+ termfilter.updateFooter()
+ event = eventHandler.waitEvent(0.25)
+
+ if event is None:
+ if main.shutdown > 1:
+ break
+ continue
+
+ helper.eventHandler(event)
+
+ if isinstance(event, bb.runqueue.runQueueExitWait):
+ if not main.shutdown:
+ main.shutdown = 1
+
+ if isinstance(event, bb.build.TaskStarted):
+ wbhbhelper.store_started_task(event, uuid)
+
+ if isinstance(event, (bb.build.TaskSucceeded,
bb.build.TaskFailedSilent, bb.build.TaskFailed)):
+ wbhbhelper.update_stored_tasks(event, uuid)
+
+ if isinstance(event, bb.event.LogExecTTY):
+ if log_exec_tty:
+ tries = event.retries
+ while tries:
+ print("Trying to run: %s" % event.prog)
+ if os.system(event.prog) == 0:
+ break
+ time.sleep(event.sleep_delay)
+ tries -= 1
+ if tries:
+ continue
+ logger.warn(event.msg)
+ continue
+
+ if isinstance(event, logging.LogRecord):
+ if event.levelno >= format.ERROR:
+ errors = errors + 1
+ return_value = 1
+ elif event.levelno == format.WARNING:
+ warnings = warnings + 1
+ # For "normal" logging conditions, don't show note logs from
tasks
+ # but do show them if the user has changed the default log
level to
+ # include verbose/debug messages
+ if event.taskpid != 0 and event.levelno <= format.NOTE:
+ continue
+ logger.handle(event)
+ continue
+
+ if isinstance(event, bb.build.TaskFailed):
+ return_value = 1
+ logfile = event.logfile
+ if logfile and os.path.exists(logfile):
+ termfilter.clearFooter()
+ bb.error("Logfile of failure stored in: %s" % logfile)
+ if includelogs and not event.errprinted:
+ print("Log data follows:")
+ f = open(logfile, "r")
+ lines = []
+ while True:
+ l = f.readline()
+ if l == '':
+ break
+ l = l.rstrip()
+ if loglines:
+ lines.append(' | %s' % l)
+ if len(lines) > int(loglines):
+ lines.pop(0)
+ else:
+ print('| %s' % l)
+ f.close()
+ if lines:
+ for line in lines:
+ print(line)
+ if isinstance(event, bb.build.TaskBase):
+ logger.info(event._message)
+ continue
+ if isinstance(event, bb.event.ParseStarted):
+ if event.total == 0:
+ continue
+ parseprogress = new_progress("Parsing recipes",
event.total).start()
+ continue
+ if isinstance(event, bb.event.ParseProgress):
+ parseprogress.update(event.current)
+ continue
+ if isinstance(event, bb.event.ParseCompleted):
+ if not parseprogress:
+ continue
+
+ parseprogress.finish()
+ print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d
targets, %d skipped, %d masked, %d errors."
+ % ( event.total, event.cached, event.parsed,
event.virtuals, event.skipped, event.masked, event.errors)))
+ continue
+
+ if isinstance(event, bb.event.CacheLoadStarted):
+ cacheprogress = new_progress("Loading cache",
event.total).start()
+ continue
+ if isinstance(event, bb.event.CacheLoadProgress):
+ cacheprogress.update(event.current)
+ continue
+ if isinstance(event, bb.event.CacheLoadCompleted):
+ cacheprogress.finish()
+ print("Loaded %d entries from dependency cache." %
event.num_entries)
+ continue
+
+ if isinstance(event, bb.command.CommandFailed):
+ return_value = event.exitcode
+ errors = errors + 1
+ logger.error("Command execution failed: %s", event.error)
+ main.shutdown = 2
+ continue
+ if isinstance(event, bb.command.CommandExit):
+ if not return_value:
+ return_value = event.exitcode
+ continue
+ if isinstance(event, (bb.command.CommandCompleted,
bb.cooker.CookerExit)):
+ main.shutdown = 2
+ continue
+ if isinstance(event, bb.event.MultipleProviders):
+ logger.info("multiple providers are available for %s%s (%s)", event._is_runtime
and "runtime " or "",
+ event._item,
+ ", ".join(event._candidates))
+ logger.info("consider defining a PREFERRED_PROVIDER entry to match
%s", event._item)
+ continue
+ if isinstance(event, bb.event.NoProvider):
+ return_value = 1
+ errors = errors + 1
+ if event._runtime:
+ r = "R"
+ else:
+ r = ""
+
+ if event._dependees:
+ logger.error("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise
requires it)", r, event._item, ", ".join(event._dependees), r)
+ else:
+ logger.error("Nothing %sPROVIDES '%s'", r, event._item)
+ if event._reasons:
+ for reason in event._reasons:
+ logger.error("%s", reason)
+ continue
+
+ if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+ logger.info("Running setscene task %d of %d (%s)" %
(event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total,
event.taskstring))
+ continue
+
+ if isinstance(event, bb.runqueue.runQueueTaskStarted):
+ if event.noexec:
+ tasktype = 'noexec task'
+ else:
+ tasktype = 'task'
+ logger.info("Running %s %s of %s (ID: %s, %s)",
+ tasktype,
+ event.stats.completed + event.stats.active +
+ event.stats.failed + 1,
+ event.stats.total, event.taskid, event.taskstring)
+ continue
+
+ if isinstance(event, bb.runqueue.runQueueTaskFailed):
+ taskfailures.append(event.taskstring)
+ logger.error("Task %s (%s) failed with exit code '%s'",
+ event.taskid, event.taskstring, event.exitcode)
+ continue
+
+ if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
+ logger.warn("Setscene task %s (%s) failed with exit code '%s' -
real task will be run instead",
+ event.taskid, event.taskstring, event.exitcode)
+ continue
+
+ if isinstance(event, bb.event.ConfigParsed):
+ # timestamp should be added for this
+ continue
+
+ if isinstance(event, bb.event.RecipeParsed):
+ # timestamp should be added for this
+ continue
+
+ if isinstance(event, bb.event.OperationStarted):
+ # timestamp should be added for this
+ continue
+
+ if isinstance(event, bb.event.OperationCompleted):
+ # timestamp should be added for this
+ # signal a complete operation
+ # calculate timing
+ continue
+
+ if isinstance(event, bb.event.DiskFull):
+ # trigger an error
+ continue
+
+ # ignore
+ if isinstance(event, (bb.event.BuildBase,
+ bb.event.StampUpdate,
+ bb.event.RecipePreFinalise,
+ bb.runqueue.runQueueEvent,
+ bb.runqueue.runQueueExitWait,
+ bb.event.OperationProgress)):
+ continue
+
+ logger.error("Unknown event: %s", event)
+
+ except EnvironmentError as ioerror:
+ termfilter.clearFooter()
+ # ignore interrupted io
+ if ioerror.args[0] == 4:
+ pass
+ except KeyboardInterrupt:
+ termfilter.clearFooter()
+ if main.shutdown == 1:
+ print("\nSecond Keyboard Interrupt, stopping...\n")
+ _, error = server.runCommand(["stateStop"])
+ if error:
+ logger.error("Unable to cleanly stop: %s" % error)
+ if main.shutdown == 0:
+ print("\nKeyboard Interrupt, closing down...\n")
+ interrupted = True
+ _, error = server.runCommand(["stateShutdown"])
+ if error:
+ logger.error("Unable to cleanly shutdown: %s" % error)
+ main.shutdown = main.shutdown + 1
+ pass
+
+ summary = ""
+ if taskfailures:
+ summary += pluralise("\nSummary: %s task failed:",
+ "\nSummary: %s tasks failed:", len(taskfailures))
+ for failure in taskfailures:
+ summary += "\n %s" % failure
+ if warnings:
+ summary += pluralise("\nSummary: There was %s WARNING message shown.",
+ "\nSummary: There were %s WARNING messages
shown.", warnings)
+ if return_value:
+ summary += pluralise("\nSummary: There was %s ERROR message shown,
returning a non-zero exit code.",
+ "\nSummary: There were %s ERROR messages shown,
returning a non-zero exit code.", errors)
+ if summary:
+ print(summary)
+
+ if interrupted:
+ print("Execution was interrupted, returning a non-zero exit code.")
+ if return_value == 0:
+ return_value = 1
+
+ termfilter.finish()
+ wbhbhelper.write_in_database(wbhbhelper.tasks_information)
+
+ return return_value
+
--
1.8.1.2
_______________________________________________
webhob mailing list
[email protected]
https://lists.yoctoproject.org/listinfo/webhob
