Ok, Leo found that the svn:eol-style wasnt set ... cheers Jan
Index: pygump/python/gump/test/testPluginBuilder.py =================================================================== --- pygump/python/gump/test/testPluginBuilder.py (Revision 220093) +++ pygump/python/gump/test/testPluginBuilder.py (Arbeitskopie) @@ -1,88 +1,91 @@ -#!/usr/bin/env python - -# Copyright 2004-2005 The Apache Software Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation" -__license__ = "http://www.apache.org/licenses/LICENSE-2.0"; - -import unittest -from unittest import TestCase - -from tempfile import mkdtemp -import stat -import os -import sys -from os import mkdir -from os import makedirs -from os.path import abspath -from os.path import isdir -from os.path import join -from shutil import rmtree -from pmock import * - -from gump.plugins.builder import ScriptBuilderPlugin - -from gump.model import Workspace, Repository, Module, Project, Script, Error - -class BuilderTestCase(MockTestCase): - def test_do_script(self): - basedir = abspath(mkdtemp()) - plugin = False - w = None - try: - wd = join(basedir,"w") - mkdir(wd) - w = Workspace("w", wd) - r = Repository(w,"r") - mkdir(join(basedir,w.name,r.name)) - m = Module(r,"m") - mpath = join(basedir,w.name,r.name,m.name) - mkdir(mpath) - p = Project(m,"p") - p.env = os.environ - if sys.platform == "win32": - scriptpath = join(mpath,"dobuild.bat") - scriptfile = open(scriptpath, mode='w') - scriptfile.write("""echo off -echo RESULT -""") - scriptfile.close() - else: - scriptpath = join(mpath,"dobuild") - scriptfile = open(scriptpath, mode='w') - scriptfile.write("""#!/bin/sh - -echo RESULT -""") - scriptfile.close() - os.chmod(scriptpath, 0755) - - log = self.mock() - log.stubs().method("debug") - plugin = ScriptBuilderPlugin(log) - plugin.initialize() - - cmd = Script(p, "dobuild") - plugin._do_script(cmd.project, cmd) - self.assertEqual("RESULT\n", cmd.build_log) - self.assertEqual(0, cmd.build_exit_status) - - cmd = Script(p, "nosuchscript") - self.assertRaises(Error, plugin._do_script, cmd.project, cmd) - finally: - if plugin: - try: plugin.finalize(w) - except: pass - rmtree(basedir) +#!/usr/bin/env python + +# Copyright 2004-2005 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation" +__license__ = "http://www.apache.org/licenses/LICENSE-2.0"; + +import unittest +from unittest import TestCase + +from tempfile import mkdtemp +import stat +import os +import sys +from os import mkdir +from os import makedirs +from os.path import abspath +from os.path import isdir +from os.path import join +from shutil import rmtree +from pmock import * + +from gump.plugins.builder import ScriptBuilderPlugin + +from gump.model import Workspace, Repository, Module, Project, Script, Error + +class BuilderTestCase(MockTestCase): + def test_do_script(self): + basedir = abspath(mkdtemp()) + plugin = False + w = None + try: + wd = join(basedir,"w") + mkdir(wd) + w = Workspace("w", wd) + r = Repository(w,"r") + mkdir(join(basedir,w.name,r.name)) + m = Module(r,"m") + mpath = join(basedir,w.name,r.name,m.name) + mkdir(mpath) + p = Project(m,"p") + p.env = os.environ + if sys.platform == "win32": + scriptpath = join(mpath,"dobuild.bat") + scriptfile = open(scriptpath, mode='w') + scriptfile.write("""@echo off +echo RESULT +""") + scriptfile.close() + else: + scriptpath = join(mpath,"dobuild") + scriptfile = open(scriptpath, mode='w') + scriptfile.write("""#!/bin/sh + +echo RESULT +""") + scriptfile.close() + os.chmod(scriptpath, 0755) + + log = self.mock() + log.stubs().method("debug") + plugin = ScriptBuilderPlugin(log) + plugin.initialize() + + cmd = Script(p, "dobuild") + plugin._do_script(cmd.project, cmd) + if sys.platform == "win32": + self.assertEqual("RESULT\r\n", cmd.build_log) + else: + self.assertEqual("RESULT\n", cmd.build_log) + self.assertEqual(0, cmd.build_exit_status) + + cmd = Script(p, "nosuchscript") + self.assertRaises(Error, plugin._do_script, cmd.project, cmd) + finally: + if plugin: + try: plugin.finalize(w) + except: pass + rmtree(basedir) \ No newline at end of file Index: pygump/python/gump/plugins/logreporter.py =================================================================== --- pygump/python/gump/plugins/logreporter.py (Revision 220093) +++ pygump/python/gump/plugins/logreporter.py (Arbeitskopie) @@ -63,8 +63,10 @@ #self.log.debug(name) for attribute in dir(object): if attribute.endswith("_log"): + #TODO unicode safety!!! logmsg = getattr(object,attribute) - self.log.debug("---%s.%s--:\n%s" % (name, attribute, logmsg)) + self.log.debug("---%s.%s--:\n" % (name, attribute)) + self.log.debug(logmsg) self.log.debug('------------------------------------------------') #else: # self.log.debug(' %s -> %s' % (attribute,getattr(object,attribute))) Index: pygump/python/gump/plugins/builder.py =================================================================== --- pygump/python/gump/plugins/builder.py (Revision 220093) +++ pygump/python/gump/plugins/builder.py (Arbeitskopie) @@ -1,191 +1,197 @@ -#!/usr/bin/env python - -# Copyright 2004-2005 The Apache Software Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Provides an abstract plugin for Command processing with subclasses.""" - -__copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation" -__license__ = "http://www.apache.org/licenses/LICENSE-2.0"; - -import os -import sys -from os.path import abspath, join, isfile -from tempfile import mkdtemp -import shutil - -from gump.model import Script, SpecificScript, Error, Project, Ant, Dependency -from gump.model.util import get_project_directory, calculate_path -from gump.plugins import AbstractPlugin -from gump.util.executor import Popen, PIPE, STDOUT -from gump.util import ansicolor - -#DEFAULT_SCRIPT_SHELL = "sh" -if sys.platform == "win32": - DEFAULT_SCRIPT_SHELL = "cmd" -else: - DEFAULT_SCRIPT_SHELL = "sh" - -class BuilderPlugin(AbstractPlugin): - """Abstract class for creating plugins that handle the execution of Commands. - - To create a subclass, override __init__, then call it from the subclass with - the method to call.""" - def __init__(self, log, cmd_clazz, method): - """Create a new builder. Arguments: - - -- log = the Logger instance for debugging - -- cmd_clazz = the Command subclass the plugin handles - -- method = the python method (which must be a class method on - the subclass) to call. It will be provided with a Project - instance and with the relevant Command instance to process""" - self.log = log - self.cmd_clazz = cmd_clazz - self.method = method - - def initialize(self): - self.tempdir = mkdtemp() - - def visit_project(self, project): - """Dispatch for each matching command (matching by class type) """ - assert isinstance(project, Project) - #self.log.debug("Visit %s looking for %s" % (project,self.cmd_clazz)) - for command in [command for command in project.commands if isinstance(command,self.cmd_clazz)]: - self.log.debug("Perform %s on %s" % (command, project)) - self.method(project, command) - - def _do_run_command(self, command, args, workdir, shell=False, no_cleanup=False): - """Utility method for actually executing commands and storing their - results within the model. - - Arguments: - - command -- the model object instance (subclass of Command) this - action is associated with - - args -- the action to take (including, for example, a script - name) - """ - # see gump.plugins.java.builder.AntPlugin for information on the - # no_cleanup flag - - # running subprocess.Popen with shell=True results in "sh -c", which is - # not what we want, since our shell=True indicates we're actually running - # a shell script, and potentially using a different shell! - if shell: - myargs = ["/usr/bin/env", command.shell or "sh"] - myargs.extend(args) - else: - myargs = args - - # unfortunately we can't use the communicate() method on the command - # it seems that, when invoking python-in-bash-in-python-in-bash (eg - # using Gump to run gump, for example) and similar complex setups, - # deadlocking can occur, for example when calling select.select(). So - # we send output to a temporary file. We can't use the regular "tmpfile" - # because when we close that file it is removed. Hence, we resort to - # using a temporary directory. *sigh* - outputfilename = os.path.join(self.tempdir, "BuilderPlugin_%s.tmp-out" % command.project.name) - outputfile = None - try: - outputfile = open(outputfilename,'wb') - cmd = Popen(myargs,shell=False,cwd=workdir,stdout=outputfile,stderr=STDOUT,env=command.env, no_cleanup=no_cleanup) - #command.build_log = cmd.communicate()[0] - command.build_exit_status = cmd.wait() - - outputfile.close() - outputfile = open(outputfilename,'rb') - # we need to avoid Unicode errors when people put in 'fancy characters' - # into build outputs - command.build_log = unicode(outputfile.read(), 'iso-8859-1') - finally: - if outputfile: - try: outputfile.close() - except: pass - - try: os.remove(outputfilename) - except: pass - - def finalize(self, workspace): - try: shutil.rmtree(self.tempdir) - except: pass - - -class PathPlugin(BuilderPlugin): - """Generate the PATH to be used with the specified command.""" - def __init__(self, log, CommandClazz): - BuilderPlugin.__init__(self, log, CommandClazz, self.set_path) - - def set_path(self, project, command): - path = calculate_path(project) - command.path = path - - -class ScriptBuilderPlugin(BuilderPlugin): - """Execute all "script" commands for all projects.""" - def __init__(self, log): - BuilderPlugin.__init__(self, log, Script, self._do_script) - - def _do_script(self, project, script): - if isinstance(script, SpecificScript): - self._do_specific_script(project, script) - return - - # environment - if script.path: - script.env['PATH'] = script.path - self.log.debug(" PATH is '%s%s%s'" % \ - (ansicolor.Blue, script.env['PATH'], ansicolor.Black)) - - # working directory - projectpath = get_project_directory(project) - if script.basedir: - projectpath = os.path.join(projectpath, script.basedir) - - # command line - myargs = [] - scriptfile = abspath(join(projectpath, script.name)) - - # No extension is ok, otherwise guess at one, platform appropriately - if not isfile(scriptfile): - oldscriptfile = scriptfile - if sys.platform == "win32": - scriptfile += ".bat" - else: - scriptfile += ".sh" - - if not isfile(scriptfile): - raise Error, "No script '%s' found!" % oldscriptfile - - myargs.append(scriptfile) - myargs.extend(script.args) - # run it - self._do_run_command(script, myargs, projectpath, shell=True) - - def _do_specific_script(self, project, script): - # environment - if script.path: - script.env['PATH'] = script.path - self.log.debug(" PATH is '%s%s%s'" % \ - (ansicolor.Blue, script.env['PATH'], ansicolor.Black)) - - # working directory - projectpath = get_project_directory(project) - if script.basedir: - projectpath = os.path.join(projectpath, script.basedir) - - # command line - myargs = [] - myargs.append(script.name) - myargs.extend(script.args) - # run it - self._do_run_command(script, myargs, projectpath, shell=False) +#!/usr/bin/env python + +# Copyright 2004-2005 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides an abstract plugin for Command processing with subclasses.""" + +__copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation" +__license__ = "http://www.apache.org/licenses/LICENSE-2.0"; + +import os +import sys +from os.path import abspath, join, isfile +from tempfile import mkdtemp +import shutil + +from gump.model import Script, SpecificScript, Error, Project, Ant, Dependency +from gump.model.util import get_project_directory, calculate_path +from gump.plugins import AbstractPlugin +from gump.util.executor import Popen, PIPE, STDOUT +from gump.util import ansicolor + +#DEFAULT_SCRIPT_SHELL = "sh" +if sys.platform == "win32": + DEFAULT_SCRIPT_SHELL = "cmd" +else: + DEFAULT_SCRIPT_SHELL = "sh" + +class BuilderPlugin(AbstractPlugin): + """Abstract class for creating plugins that handle the execution of Commands. + + To create a subclass, override __init__, then call it from the subclass with + the method to call.""" + def __init__(self, log, cmd_clazz, method): + """Create a new builder. Arguments: + + -- log = the Logger instance for debugging + -- cmd_clazz = the Command subclass the plugin handles + -- method = the python method (which must be a class method on + the subclass) to call. It will be provided with a Project + instance and with the relevant Command instance to process""" + self.log = log + self.cmd_clazz = cmd_clazz + self.method = method + + def initialize(self): + self.tempdir = mkdtemp() + + def visit_project(self, project): + """Dispatch for each matching command (matching by class type) """ + assert isinstance(project, Project) + #self.log.debug("Visit %s looking for %s" % (project,self.cmd_clazz)) + for command in [command for command in project.commands if isinstance(command,self.cmd_clazz)]: + self.log.debug("Perform %s on %s" % (command, project)) + self.method(project, command) + + def _do_run_command(self, command, args, workdir, shell=False, no_cleanup=False): + """Utility method for actually executing commands and storing their + results within the model. + + Arguments: + - command -- the model object instance (subclass of Command) this + action is associated with + - args -- the action to take (including, for example, a script + name) + """ + # see gump.plugins.java.builder.AntPlugin for information on the + # no_cleanup flag + + # running subprocess.Popen with shell=True results in "sh -c", which is + # not what we want, since our shell=True indicates we're actually running + # a shell script, and potentially using a different shell! + if sys.platform != "win32" and shell: + myargs = ["/usr/bin/env", command.shell or "sh"] + myargs.extend(args) + else: + myargs = args + + # unfortunately we can't use the communicate() method on the command + # it seems that, when invoking python-in-bash-in-python-in-bash (eg + # using Gump to run gump, for example) and similar complex setups, + # deadlocking can occur, for example when calling select.select(). So + # we send output to a temporary file. We can't use the regular "tmpfile" + # because when we close that file it is removed. Hence, we resort to + # using a temporary directory. *sigh* + outputfilename = os.path.join(self.tempdir, "BuilderPlugin_%s.tmp-out" % command.project.name) + outputfile = None + try: + outputfile = open(outputfilename,'wb') + + # in case there is weird stuff in the env dictionary we don't want + # that to result in an error inside subprocess, so explicitly convert + # everything to strings + for k,v in command.env.iteritems(): + command.env[k] = str(v) + cmd = Popen(myargs,shell=False,cwd=workdir,stdout=outputfile,stderr=STDOUT,env=command.env, no_cleanup=no_cleanup) + #command.build_log = cmd.communicate()[0] + command.build_exit_status = cmd.wait() + + outputfile.close() + outputfile = open(outputfilename,'rb') + # we need to avoid Unicode errors when people put in 'fancy characters' + # into build outputs + command.build_log = unicode(outputfile.read(), 'iso-8859-1') + finally: + if outputfile: + try: outputfile.close() + except: pass + + try: os.remove(outputfilename) + except: pass + + def finalize(self, workspace): + try: shutil.rmtree(self.tempdir) + except: pass + + +class PathPlugin(BuilderPlugin): + """Generate the PATH to be used with the specified command.""" + def __init__(self, log, CommandClazz): + BuilderPlugin.__init__(self, log, CommandClazz, self.set_path) + + def set_path(self, project, command): + path = calculate_path(project) + command.path = path + + +class ScriptBuilderPlugin(BuilderPlugin): + """Execute all "script" commands for all projects.""" + def __init__(self, log): + BuilderPlugin.__init__(self, log, Script, self._do_script) + + def _do_script(self, project, script): + if isinstance(script, SpecificScript): + self._do_specific_script(project, script) + return + + # environment + if script.path: + script.env['PATH'] = script.path + self.log.debug(" PATH is '%s%s%s'" % \ + (ansicolor.Blue, script.env['PATH'], ansicolor.Black)) + + # working directory + projectpath = get_project_directory(project) + if script.basedir: + projectpath = os.path.join(projectpath, script.basedir) + + # command line + myargs = [] + scriptfile = abspath(join(projectpath, script.name)) + + # No extension is ok, otherwise guess at one, platform appropriately + if not isfile(scriptfile): + oldscriptfile = scriptfile + if sys.platform == "win32": + scriptfile += ".bat" + else: + scriptfile += ".sh" + + if not isfile(scriptfile): + raise Error, "No script '%s' found!" % oldscriptfile + + myargs.append(scriptfile) + myargs.extend(script.args) + # run it + self._do_run_command(script, myargs, projectpath, shell=True) + + def _do_specific_script(self, project, script): + # environment + if script.path: + script.env['PATH'] = script.path + self.log.debug(" PATH is '%s%s%s'" % \ + (ansicolor.Blue, script.env['PATH'], ansicolor.Black)) + + # working directory + projectpath = get_project_directory(project) + if script.basedir: + projectpath = os.path.join(projectpath, script.basedir) + + # command line + myargs = [] + myargs.append(script.name) + myargs.extend(script.args) + # run it + self._do_run_command(script, myargs, projectpath, shell=False) \ No newline at end of file Index: pygump/python/gump/util/executor.py =================================================================== --- pygump/python/gump/util/executor.py (Revision 220093) +++ pygump/python/gump/util/executor.py (Arbeitskopie) @@ -5,9 +5,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,7 +49,8 @@ from subprocess import PIPE from subprocess import STDOUT import subprocess - + import os + class Popen(subprocess.Popen): """This is a thin wrapper around subprocess.Popen which does fancy logging.""" def __init__(self, args, bufsize=0, executable=None, @@ -99,7 +100,7 @@ if f: try: f.close() except: pass - + class Popen(subprocess.Popen): """This is a thin wrapper around subprocess.Popen which handles process group management. The gump.util.executor.clean_up_processes() @@ -130,8 +131,8 @@ # call in there while still supporting the originally provided # function pre_exec_function = lambda: (preexec_fn(),savepgid(process_list_filename)) - - + + subprocess.Popen.__init__(self, args, bufsize=bufsize, executable=executable, stdin=stdin, stdout=stdout, stderr=stderr, # note our custom function in there... @@ -145,12 +146,12 @@ preexec_fn=preexec_fn, close_fds=close_fds, shell=shell, cwd=cwd, env=env, universal_newlines=universal_newlines, startupinfo=startupinfo, creationflags=creationflags) - + def clean_up_processes(timeout=300): """This function can be called prior to program exit to attempt to kill all our running children that were created using this module.""" - + global process_list_filename global temp_dir @@ -161,7 +162,7 @@ f = open(process_list_filename, 'r') pgrp_list = [int(line) for line in f.read().splitlines()] except: - if f: + if f: try: f.close() except: pass try: @@ -169,11 +170,11 @@ shutil.rmtree(temp_dir) except: pass - + # send SIGTERM to everything, and update pgrp_list to just those # process groups which have processes in them. _kill_groups(pgrp_list, signal.SIGTERM) - + # pass a copy of the process groups. we want to remember every # group that we SIGTERM'd so that we can SIGKILL them later. it # is possible that a process in the pgrp was reparented to the @@ -181,17 +182,17 @@ # want to mistakenly think we've killed all processes in the # group. thus, we preserve the list and SIGKILL it later. _reap_children(pgrp_list[:], timeout) - + # SIGKILL everything, editing pgrp_list again. _kill_groups(pgrp_list, signal.SIGKILL) - + # reap everything left, but don't really bother waiting on them. # if we exit, then init will reap them. _reap_children(pgrp_list, 60) def _kill_groups(pgrp_list, sig): # NOTE: this function edits pgrp_list - + for pgrp in pgrp_list[:]: try: os.killpg(pgrp, sig) @@ -201,21 +202,21 @@ def _reap_children(pgrp_list, timeout): # NOTE: this function edits pgrp_list - + # keep reaping until the timeout expires, or we finish end_time = time.time() + timeout - + # keep reaping until all pgrps are done, or we run out of time while pgrp_list and time.time() < end_time: # if there's no groups left, we're done, so let's # exit early! if len(pgrp_list) == 0: break - + # pause for a bit while processes work on exiting. this pause is # at the top, so we can also pause right after the killpg() time.sleep(1) - + # go through all pgrps to reap them for pgrp in pgrp_list[:]: # loop quickly to clean everything in this pgrp @@ -231,4 +232,4 @@ if pid == 0: # some stuff has not exited yet, and WNOHANG avoided # blocking. go ahead and move to the next pgrp. - break + break \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [EMAIL PROTECTED] For additional commands, e-mail: [EMAIL PROTECTED]
