Changeset: d13c8a625130 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d13c8a625130
Removed Files:
testing/subprocess26.py
Modified Files:
testing/Makefile.ag
testing/Mtest.py.in
Branch: Feb2013
Log Message:
Use our own process module instead of old subprocess26 and buffer output.
When launching the server, use a _BufferedOutput class from our own
process module to read output as it comes (in a separate thread).
This in order to avoid deadlock when the server produces lots of
output.
Also we now don't use subprocess26 anymore. Assume all systems have
finally upgraded to Python >= 2.6.
diffs (truncated from 1502 to 300 lines):
diff --git a/testing/Makefile.ag b/testing/Makefile.ag
--- a/testing/Makefile.ag
+++ b/testing/Makefile.ag
@@ -45,7 +45,7 @@ scripts_py = {
headers_python = {
HEADERS = py
DIR = $(prefix)/$(PYTHON2_LIBDIR)/MonetDBtesting
- SOURCES = trace.py process.py monet_options.py.in __init__.py
subprocess26.py listexports.py.in
+ SOURCES = trace.py process.py monet_options.py.in __init__.py
listexports.py.in
}
scripts_sh = {
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -138,15 +138,16 @@ def _configure(str):
str = nstr
return str
+# use our own process module because it has _BufferedPipe
try:
- import subprocess26 as subprocess
+ import process
except ImportError:
try:
- import MonetDBtesting.subprocess26 as subprocess
+ import MonetDBtesting.process as process
except ImportError:
p = _configure(os.path.join('@QXprefix@', '@PYTHON2_LIBDIR@'))
sys.path.insert(0, p)
- import MonetDBtesting.subprocess26 as subprocess
+ import MonetDBtesting.process as process
if os.environ.has_key('PYTHONPATH'):
p += os.pathsep + os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = p
@@ -157,8 +158,8 @@ if isatty and os.isatty(sys.stdin.fileno
ttywidth = 80
else:
try:
- proc = subprocess.Popen(['stty', '-a'], stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
+ proc = process.Popen(['stty', '-a'], stdout = process.PIPE,
+ stderr = process.PIPE)
except OSError:
pass
else:
@@ -1306,9 +1307,9 @@ def ApproveOutput (env, TST) :
patch = os.path.join(os.sep, 'tmp')
patch = os.path.join(patch, "%s.patch-%s" %
(os.path.basename(stableOUTPUT), str(os.getpid())))
f = open(patch + '.0', 'w')
- proc = subprocess.Popen(['diff', '-Bb', '-I^[#=]', '-U0',
- stableOUTPUT, testOUTPUT],
- stdout = f)
+ proc = process.Popen(['diff', '-Bb', '-I^[#=]', '-U0',
+ stableOUTPUT, testOUTPUT],
+ stdout = f)
proc.wait()
f.close()
if os.path.getsize(patch + ".0"):
@@ -1335,10 +1336,12 @@ def ApproveOutput (env, TST) :
f.flush()
f.close()
shutil.copy(stableOUTPUT, stableOUTPUT + ".ORG")
- proc = subprocess.Popen(['patch', stableOUTPUT, patch + '.1'])
+ proc = process.Popen(['patch', stableOUTPUT, patch + '.1'])
proc.wait()
f = open(patch, 'w')
- proc = subprocess.Popen(['diff', '-u', stableOUTPUT + '.ORG',
stableOUTPUT], stdout = f)
+ proc = process.Popen(['diff', '-u', stableOUTPUT + '.ORG',
+ stableOUTPUT],
+ stdout = f)
proc.wait()
f.close()
remove(stableOUTPUT + ".ORG")
@@ -1471,7 +1474,8 @@ def GetBitsAndOIDsAndModsAndStaticAndThr
cmd.append('--dbpath=%s' % os.path.join(env['GDK_DBFARM'], TSTPREF))
if procdebug:
print 'GetBitsAndOIDsAndModsAndStaticAndThreads: starting process "%s"
(inpipe, outpipe, errpipe)\n' % '" "'.join(cmd)
- proc = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout =
subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
+ proc = process.Popen(cmd, stdin = process.PIPE, stdout = process.PIPE,
+ stderr = process.PIPE, universal_newlines = True)
proc.killed = False
t = Timer(float(par['TIMEOUT']), killProc, args = [proc, proc.stderr, cmd])
try:
@@ -2170,7 +2174,8 @@ def RunTest(env, TST, BusyPorts, COND, o
'%s.out.diff.html' % TST])
if procdebug:
print 'RunTest: starting process "%s"\n' % '" "'.join(cmd)
- proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr =
subprocess.PIPE)
+ proc = process.Popen(cmd, stdout = process.PIPE,
+ stderr = process.PIPE)
proc.killed = False
t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
try:
@@ -2203,7 +2208,7 @@ def RunTest(env, TST, BusyPorts, COND, o
'-U%s' % par['CONTEXT'],
'%s%s.FILTERED' % (TST, STABLEout),
'%s.test.out.FILTERED' % TST])
- proc = subprocess.Popen(cmd)
+ proc = process.Popen(cmd)
proc.wait()
diff_html = open('%s.err.diff.html' % TST,"w")
@@ -2239,7 +2244,8 @@ def RunTest(env, TST, BusyPorts, COND, o
'%s.err.diff.html' % TST])
if procdebug:
print 'RunTest: starting process "%s"\n' % '" "'.join(cmd)
- proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr =
subprocess.PIPE)
+ proc = process.Popen(cmd, stdout = process.PIPE,
+ stderr = process.PIPE)
proc.killed = False
t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
try:
@@ -2272,7 +2278,7 @@ def RunTest(env, TST, BusyPorts, COND, o
'-U%s' % par['CONTEXT'],
'%s%s.FILTERED' % (TST, STABLEerr),
'%s.test.err.FILTERED' % TST])
- proc = subprocess.Popen(cmd)
+ proc = process.Popen(cmd)
proc.wait()
FailedOut, FailedErr = CategorizeResult(TST, max(sockerr, errcode))
@@ -2467,15 +2473,15 @@ def killProc(proc, outfile = None, cmd =
else:
cdb = None
if cdb:
- p = subprocess.Popen([cdb, '-pv', '-p', str(proc.pid),
- '-y',
'%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines',
'-c', '~*kP;!locks;q'],
- stdout = subprocess.PIPE)
+ p = process.Popen([cdb, '-pv', '-p', str(proc.pid),
+ '-y',
'%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines',
'-c', '~*kP;!locks;q'],
+ stdout = process.PIPE)
out, err = p.communicate()
else:
out = ''
else:
try:
- p = subprocess.Popen(['pstack', str(proc.pid)], stdout =
subprocess.PIPE)
+ p = process.Popen(['pstack', str(proc.pid)], stdout = process.PIPE)
out, err = p.communicate()
except:
out = ''
@@ -2490,8 +2496,8 @@ def killProc(proc, outfile = None, cmd =
except AttributeError:
if procdebug:
print 'killProc: starting process "taskkill" "/F" "/T" "/PID"
"%s"\n' % str(proc.pid)
- p = subprocess.Popen(['taskkill','/F','/T','/PID',str(proc.pid)],
- stdout = subprocess.PIPE, stderr =
subprocess.PIPE)
+ p = process.Popen(['taskkill','/F','/T','/PID',str(proc.pid)],
+ stdout = process.PIPE, stderr = process.PIPE)
out, err = p.communicate()
if procdebug:
print 'killProc: process exited "taskkill" "/F" "/T" "/PID" "%s"
(%s)\n' % (str(proc.pid), proc.returncode)
@@ -2500,7 +2506,7 @@ def killProc(proc, outfile = None, cmd =
def LaunchIt(cmd, TestInput, TestOut, TestErr, TimeOut, SrvrOut = None) :
if not SrvrOut:
- SrvrOut = subprocess.PIPE
+ SrvrOut = process.PIPE
TestOut.write(Prompt(cmd))
TestOut.flush()
@@ -2509,7 +2515,13 @@ def LaunchIt(cmd, TestInput, TestOut, Te
if procdebug:
print 'LaunchIt: starting process "%s" (inpipe)\n' % '" "'.join(cmd)
- proc = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = SrvrOut,
stderr = TestErr, universal_newlines = True)
+ proc = process.Popen(cmd, stdin = process.PIPE, stdout = SrvrOut,
+ stderr = TestErr, universal_newlines = True)
+ # maybe buffer output as it comes to avoid deadlock
+ if SrvrOut == process.PIPE:
+ proc.stdout = process._BufferedPipe(proc.stdout)
+ if TestErr == process.PIPE:
+ proc.stderr = process._BufferedPipe(proc.stderr)
proc.killed = False
t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd])
t.start()
@@ -2536,7 +2548,7 @@ def CollectIt(pOut, TestOut) :
def RunIt(cmd, TestIn, TestOut, TestErr, TimeOut) :
if type(TestIn) is type(''):
TestInput = TestIn
- TestIn = subprocess.PIPE
+ TestIn = process.PIPE
else:
TestInput = None
TestOut.write(Prompt(cmd))
@@ -2545,7 +2557,7 @@ def RunIt(cmd, TestIn, TestOut, TestErr,
TestErr.flush()
if procdebug:
print 'RunIt: starting process "%s"\n' % '" "'.join(cmd)
- proc = subprocess.Popen(cmd, stdin = TestIn, stdout = TestOut, stderr =
TestErr, universal_newlines = True)
+ proc = process.Popen(cmd, stdin = TestIn, stdout = TestOut, stderr =
TestErr, universal_newlines = True)
proc.killed = False
t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd])
try:
@@ -2922,7 +2934,8 @@ def DoIt(env, SERVER, CALL, TST, EXT, PR
def Check(command, input) :
if procdebug:
print 'Check: starting process "%s" (inpipe,outpipe,errpipe)\n' % '"
"'.join(command)
- proc = subprocess.Popen(command, stdin = subprocess.PIPE, stdout =
subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
+ proc = process.Popen(command, stdin = process.PIPE, stdout = process.PIPE,
+ stderr = process.PIPE, universal_newlines = True)
proc.killed = False
t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
try:
@@ -2963,7 +2976,10 @@ def Check(command, input) :
STDERR.flush()
#sys.exit(1)
if sys.platform == 'linux2':
- proc = subprocess.Popen(['netstat', '-ap'], stdout =
subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
+ proc = process.Popen(['netstat', '-ap'],
+ stdout = process.PIPE,
+ stderr = process.PIPE,
+ universal_newlines = True)
out, err = proc.communicate()
STDERR.write(err)
STDOUT.write(out)
@@ -3117,7 +3133,9 @@ if os.name == "nt":
r = re.compile('^Microsoft Windows (.*)\[Version
([0-9]+\.[0-9]+)([^\[0-9].*)\]$')
if procdebug:
print 'starting process "cmd" "/c" "ver" (inpipe,outpipe)\n'
- proc = subprocess.Popen('cmd /c ver', stdin = subprocess.PIPE, stdout =
subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
+ proc = process.Popen('cmd /c ver', stdin = process.PIPE,
+ stdout = process.PIPE, stderr = process.PIPE,
+ universal_newlines = True)
qOut, qErr = proc.communicate()
if procdebug:
print 'process exited "cmd" "/c" "ver" (%s)\n' % proc.returncode
@@ -3661,7 +3679,7 @@ def main(argv) :
REV = opts.get('revision')
if REV is None: # no --revision option: try to find out
try:
- proc = subprocess.Popen(['hg', 'id', '-i'], stdout =
subprocess.PIPE)
+ proc = process.Popen(['hg', 'id', '-i'], stdout = process.PIPE)
out, err = proc.communicate()
if proc.returncode == 0:
REV = out.strip()
diff --git a/testing/subprocess26.py b/testing/subprocess26.py
deleted file mode 100644
--- a/testing/subprocess26.py
+++ /dev/null
@@ -1,1260 +0,0 @@
-# subprocess - Subprocesses with accessible I/O streams
-#
-# For more information about this module, see PEP 324.
-#
-# This module should remain compatible with Python 2.2, see PEP 291.
-#
-# Copyright (c) 2003-2005 by Peter Astrand <[email protected]>
-#
-# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/2.4/license for licensing details.
-
-r"""subprocess - Subprocesses with accessible I/O streams
-
-This module allows you to spawn processes, connect to their
-input/output/error pipes, and obtain their return codes. This module
-intends to replace several other, older modules and functions, like:
-
-os.system
-os.spawn*
-os.popen*
-popen2.*
-commands.*
-
-Information about how the subprocess module can be used to replace these
-modules and functions can be found below.
-
-
-
-Using the subprocess module
-===========================
-This module defines one class called Popen:
-
-class Popen(args, bufsize=0, executable=None,
- stdin=None, stdout=None, stderr=None,
- preexec_fn=None, close_fds=False, shell=False,
- cwd=None, env=None, universal_newlines=False,
- startupinfo=None, creationflags=0):
-
-
-Arguments are:
-
-args should be a string, or a sequence of program arguments. The
-program to execute is normally the first item in the args sequence or
-string, but can be explicitly set by using the executable argument.
-
-On UNIX, with shell=False (default): In this case, the Popen class
-uses os.execvp() to execute the child program. args should normally
-be a sequence. A string will be treated as a sequence with the string
-as the only item (the program to execute).
-
-On UNIX, with shell=True: If args is a string, it specifies the
-command string to execute through the shell. If args is a sequence,
-the first item specifies the command string, and any additional items
-will be treated as additional shell arguments.
-
-On Windows: the Popen class uses CreateProcess() to execute the child
-program, which operates on strings. If args is a sequence, it will be
-converted to a string using the list2cmdline method. Please note that
_______________________________________________
checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
