Update of /cvsroot/monetdb/MonetDB/src/testing
In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv31123
Modified Files:
Tag: MonetDB_1-20
Mtest.py.in subprocess26.py
Log Message:
Do not use Mtimeout anymore for controlling the time a subprocess is
allowed to run. Use an internal timer instead.
The problem with Mtimeout is that it is not portable to Windows,
whereas using a thread is.
On Windows, in order for this to work, you do need the program psKill
in your PATH. psKill can be obtained from the Windows Sysinternals
website (http://www.microsoft.com/technet/sysinternals/default.mspx)
in both the psKill and psTools packages.
On Linux, we need to now use our own version of the subprocess module,
since we need to set the process group of the process being started so
that it can be killed together with its children.
(Another advantage of this change is that when you interrupt a test,
the children get killed. In the past, because Mtimeout also started
the children in a different process group, those children did not get
the interrupt signal, so would continue, even when Mtest.py was
killed.)
Index: Mtest.py.in
===================================================================
RCS file: /cvsroot/monetdb/MonetDB/src/testing/Mtest.py.in,v
retrieving revision 1.309.2.11
retrieving revision 1.309.2.12
diff -u -d -r1.309.2.11 -r1.309.2.12
--- Mtest.py.in 22 Oct 2007 12:13:43 -0000 1.309.2.11
+++ Mtest.py.in 22 Oct 2007 13:25:04 -0000 1.309.2.12
@@ -106,11 +106,8 @@
if os.environ.has_key('PYTHONPATH'):
p = p + os.pathsep + os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = p
-try:
- import subprocess
-except ImportError:
- # use private copy for old Python versions
- import MonetDB.subprocess26 as subprocess
+import MonetDB.subprocess26 as subprocess
+import threading, signal
randomPortRepeat = 9
@@ -1157,31 +1154,39 @@
def GetBitsAndOIDsAndModsAndStatic(env) :
rtrn = 0
TSTPREF = env['TSTPREF']
- cmd = '%s%s --dbname=%s' % (env['exe']['Mtimeout'][1],
env['exe']['Mserver'][1], TSTPREF)
+ cmd = '%s --dbname=%s' % (env['exe']['Mserver'][1], TSTPREF)
proc = subprocess.Popen(cmd, shell = True, stdin = subprocess.PIPE, stdout
= subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
- if par['M5']:
- input = '''\
- clients.quit();
- '''
- else:
- input = '''\
- help("kunion");
- {
- # print a list of all modules found
- # "Modules: mod1, mod2, ..."
- var MODs :=
kunion(bat("monet_mod_nme").reverse().mark(oid(nil)),view_modules().reverse().mark(oid(nil))).kunique();
- printf("\\nModules: ");
- var x := "";
- [EMAIL PROTECTED](){
- printf("%s\'%s\'",x,$h);
- x:=",";
+ t = threading.Timer(float(par['TIMEOUT']), killProc, args = [proc,
proc.stderr, cmd])
+ try:
+ t.start()
+ if par['M5']:
+ input = '''\
+ clients.quit();
+ '''
+ else:
+ input = '''\
+ help("kunion");
+ {
+ # print a list of all modules found
+ # "Modules: mod1, mod2, ..."
+ var MODs :=
kunion(bat("monet_mod_nme").reverse().mark(oid(nil)),view_modules().reverse().mark(oid(nil))).kunique();
+ printf("\\nModules: ");
+ var x := "";
+ [EMAIL PROTECTED](){
+ printf("%s\'%s\'",x,$h);
+ x:=",";
+ }
+ printf("\\n");
}
- printf("\\n");
- }
- quit();
- '''
- ##module("NoModule");
- qOut, qErr = proc.communicate(input = input)
+ quit();
+ '''
+ ##module("NoModule");
+ qOut, qErr = proc.communicate(input = input)
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc, proc.stderr, cmd)
+ raise
env['TST_MODS'] = []
env['TST_BITS'] = ""
env['TST_OIDS'] = ""
@@ -1280,23 +1285,31 @@
# Warn("Monet5: Check, whether required BATs do exist, is not
possible, yet!")
return missing
- cmd = '%s%s --dbname=%s' % (env['exe']['Mtimeout'][1],
env['exe']['Mserver'][1], TSTDB)
+ cmd = '%s --dbname=%s' % (env['exe']['Mserver'][1], TSTDB)
proc = subprocess.Popen(cmd, shell = True, stdin = subprocess.PIPE, stdout
= subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
- qOut, qErr = proc.communicate('''\
- {
- # print a list of all persistent BATs
- # "BATs: bat1, bat2, ..."
- var BATs := view_bbp_name();
- printf("\\nBATs: ");
- var x := "";
- [EMAIL PROTECTED](){
- printf("%s\'%s\'",x,$t);
- x:=",";
+ t = threading.Timer(float(par['TIMEOUT']), killProc, args = [proc,
proc.stderr, cmd])
+ try:
+ t.start()
+ qOut, qErr = proc.communicate('''\
+ {
+ # print a list of all persistent BATs
+ # "BATs: bat1, bat2, ..."
+ var BATs := view_bbp_name();
+ printf("\\nBATs: ");
+ var x := "";
+ [EMAIL PROTECTED](){
+ printf("%s\'%s\'",x,$t);
+ x:=",";
+ }
+ printf("\\n");
}
- printf("\\n");
- }
- quit();
- ''')
+ quit();
+ ''')
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc, proc.stderr, cmd)
+ raise
TST_BATS = []
if qOut:
tb = re.compile("^BATs: (.*)$", re.MULTILINE)
@@ -1681,7 +1694,10 @@
TestErr.close()
t0 = time.time()
- DoIt (env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile, TestErrFile,
STIMEOUT, CTIMEOUT, TIMEOUT, MkillUsersAT, ME, MAPIsockets, XRPCsockets)
+ if DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOutFile,
TestErrFile, STIMEOUT, CTIMEOUT, TIMEOUT, MkillUsersAT, ME, MAPIsockets,
XRPCsockets):
+ timeout = F_TIME
+ else:
+ timeout = F_OK
t1 = time.time()
TX = t1 - t0
if not quiet:
@@ -1734,12 +1750,6 @@
##rm -f .new.left-over.tmp.bats.
$TSTTRGBASE/Tests/.old.left-over.tmp.bats.
##if [ -f .all.left-over.tmp.bats. ] ; then mv -f
.all.left-over.tmp.bats. $TSTTRGBASE/Tests/.old.left-over.tmp.bats. ; fi
- timeout = F_OK
- mto = re.compile("^!(Mtimeout:)? Timeout:", re.MULTILINE)
- for l in open(TestErr.name):
- if mto.match(l):
- timeout = F_TIME
-
if timeout == F_TIME:
if quiet:
STDOUT.write("\n%s : Timeout!\n" % TST)
@@ -1792,11 +1802,6 @@
MDIFF0 = env['exe']['Mdiff'][1]
MDIFF1 = MDIFF0+' -d'
- if env['exe']['Mtimeout'][0]:
- if quiet:
- MDIFF1 = env['exe']['Mtimeout'][1]+' -q '+MDIFF1
- else:
- MDIFF1 = env['exe']['Mtimeout'][1]+' '+MDIFF1
#TODO:
#timedout = 1
@@ -1828,13 +1833,26 @@
# filesizes differ significantly => expect major
differences!
ACCURACYout = -1
while timedout and MDIFF == MDIFF1:
+ cmd = ['Mdiff']
if ACCURACYout == -1:
MDIFF = MDIFF0
ACCURACYout = 0
- if not quiet:
- timedout = os.system('%s -I"%s" "%s" -A%d -r"%s"
"%s%s.FILTERED" "%s.test.out.FILTERED" "%s.out.diff.html"' % (MDIFF,
par['IGNORE'], par['CONTEXT'], ACCURACYout, REVe, TST, STABLEout, TST, TST))
else:
- timedout = os.system('%s -q -I"%s" "%s" -A%d -r"%s"
"%s%s.FILTERED" "%s.test.out.FILTERED" "%s.out.diff.html"' % (MDIFF,
par['IGNORE'], par['CONTEXT'], ACCURACYout, REVe, TST, STABLEout, TST, TST))
+ cmd.append('-d')
+ if quiet:
+ cmd.append('-q')
+ proc = subprocess.Popen(cmd + ['-I%s' % par['IGNORE'],
par['CONTEXT'], '-A%d' % ACCURACYout, '-r%s' % REVe, '%s%s.FILTERED' % (TST,
STABLEout), '%s.test.out.FILTERED' % TST, '%s.out.diff.html' % TST])
+ proc.killed = False
+ t = threading.Timer(float(par['TIMEOUT']), killProc, args =
[proc])
+ try:
+ t.start()
+ proc.wait()
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc)
+ raise
+ timedout = proc.killed
ACCURACYout = ACCURACYout - 1
#TODO:
@@ -2006,6 +2024,15 @@
return prmpt+prmpt+cmd+prmpt+"\n\n"
### Prompt(cmd) #
+def killProc(proc, outfile = None, cmd = None):
+ try:
+ os.kill(-proc.pid, signal.SIGTERM)
+ except AttributeError:
+ subprocess.Popen(['pskill','-t',str(proc.pid)])
+ if outfile is not None and cmd is not None:
+ outfile.write('\n!Mtimeout: Timeout: %s\n' % cmd)
+ proc.killed = True
+
def LaunchIt(cmd, TestInput, TestOut, TestErr) :
TestOut.write(Prompt(cmd))
TestOut.flush()
@@ -2033,7 +2060,7 @@
TestOut.write(buf)
### CollectIt(pOut, pErr, TestOut, TestErr) #
-def RunIt(cmd, TestIn, TestOut, TestErr) :
+def RunIt(cmd, TestIn, TestOut, TestErr, TimeOut) :
if type(TestIn) is type(''):
TestInput = TestIn
TestIn = subprocess.PIPE
@@ -2044,9 +2071,19 @@
TestErr.write(Prompt(cmd))
TestErr.flush()
proc = subprocess.Popen(cmd, shell = True, stdin = TestIn, stdout =
TestOut, stderr = TestErr, universal_newlines = True)
- # since both stdout and stderr are redirected to files,
- # communicate will not return any useful data
- proc.communicate(input = TestInput)
+ proc.killed = False
+ t = threading.Timer(TimeOut, killProc, args = [proc, TestErr, cmd])
+ try:
+ t.start()
+ # since both stdout and stderr are redirected to files,
+ # communicate will not return any useful data
+ proc.communicate(input = TestInput)
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc, TestErr, cmd)
+ raise
+ return proc.killed
### RunIt(cmd, TestIn, TestOut, TestErr) #
def Log() :
@@ -2095,16 +2132,6 @@
Warn("Flushing STDOUT in DoIt failed with #%d: '%s'." % (IOerrNo,
IOerrStr))
TSTDB = env['TSTDB']
exe = env['exe']
- if exe['Mtimeout'][0]:
- MTO = "Mtimeout -timeout "
- TOT = TIMEOUT
- STO = STIMEOUT
- CTO = CTIMEOUT
- else:
- MTO = ""
- TOT = ""
- STO = ""
- CTO = ""
LOCAL_CONF = ""
if os.path.isfile(TST+".conf"):
@@ -2140,7 +2167,7 @@
if os.path.isfile(TST+".prologue5") and par['M5']:
PROLOGUE = " "+TST+".prologue5"
- Srvr = '%s%s %s%s "--dbname=%s"' % (MTO, str(STO), exe['Mserver'][1],
LOCAL_CONF, TSTDB)
+ Srvr = '%s%s "--dbname=%s"' % (exe['Mserver'][1], LOCAL_CONF, TSTDB)
DBINIT=""
if os.path.isfile(TST+".dbinit") and par['M4']:
@@ -2194,15 +2221,16 @@
ClntOut = open(TestOutFile, 'a')
ClntErr = open(TestErrFile, 'a')
+ killed = False
if ServerReady:
if CALL == "other":
- cmd = MTO+str(CTO)+" "+os.path.join(".",TST+EXT)+" "+TST+"
"+PRELUDE
- RunIt(cmd, "", ClntOut, ClntErr)
+ cmd = os.path.join(".",TST+EXT)+" "+TST+" "+PRELUDE
+ killed = RunIt(cmd, "", ClntOut, ClntErr, CTIMEOUT)
elif CALL == "python":
- cmd = MTO+str(CTO)+" "+exe['python'][1]+" "+TST+EXT+" "+TST+"
"+PRELUDE
- RunIt(cmd, "", ClntOut, ClntErr)
+ cmd = exe['python'][1]+" "+TST+EXT+" "+TST+" "+PRELUDE
+ killed = RunIt(cmd, "", ClntOut, ClntErr, CTIMEOUT)
elif CALL in ["mal", "malXs", "milS", "milSXs"]:
- cmd = '%s%s %s%s --dbname=%s %s ' % (MTO, str(TOT),
exe['Mserver'][1], LOCAL_CONF, TSTDB, PRELUDE)
+ cmd = '%s%s --dbname=%s %s ' % (exe['Mserver'][1], LOCAL_CONF,
TSTDB, PRELUDE)
if CALL in ["mal", "milS"]:
X=""
else:
@@ -2213,11 +2241,13 @@
for f in d:
if test.match(f):
if CALL == "mal":
- RunIt(cmd + f, open(devnull), ClntOut, ClntErr)
+ killed = RunIt(cmd + f, open(devnull), ClntOut,
ClntErr, TIMEOUT)
elif par['M5']:
- RunIt(cmd+" --dbinit=\"include mil_scenario; mil();\"
"+f, open(devnull), ClntOut, ClntErr)
+ killed = RunIt(cmd+" --dbinit=\"include mil_scenario;
mil();\" "+f, open(devnull), ClntOut, ClntErr, TIMEOUT)
else:
- RunIt(cmd, open(f), ClntOut, ClntErr)
+ killed = RunIt(cmd, open(f), ClntOut, ClntErr, TIMEOUT)
+ if killed:
+ break
elif CALL in ["milC", "milCXs", "malC", "malCXs"]:
TSTs = []
if CALL in ("milC", "malC"):
@@ -2231,13 +2261,16 @@
if test.match(f):
TSTs.append(f)
- Clnt = MTO+str(TOT)+" "
if CALL[:3] == "mil":
- Clnt = Clnt+exe['MIL_Client'][1]
- if CALL[:3] == "mal":
- Clnt = Clnt+exe['MAL_Client'][1]
+ Clnt = exe['MIL_Client'][1]
+ elif CALL[:3] == "mal":
+ Clnt = exe['MAL_Client'][1]
+ else:
+ Clnt = '' # cannot happen
for f in TSTs:
- RunIt(Clnt, open(f), ClntOut, ClntErr)
+ killed = RunIt(Clnt, open(f), ClntOut, ClntErr, TIMEOUT)
+ if killed:
+ break
#TODO
#elif CALL == "milCXp":
@@ -2254,9 +2287,11 @@
if test.match(f):
TSTs.append(f)
- Clnt = MTO+str(TOT)+" "+exe['SQL_Client'][1]
+ Clnt = exe['SQL_Client'][1]
for f in TSTs:
- RunIt(Clnt, open(f), ClntOut, ClntErr)
+ killed = RunIt(Clnt, open(f), ClntOut, ClntErr, TIMEOUT)
+ if killed:
+ break
elif CALL in ["xq", "xqXs"]:
TSTs = []
if CALL == "xq":
@@ -2270,9 +2305,11 @@
if test.match(f):
TSTs.append(f)
- Clnt = MTO+str(TOT)+" "+exe['XQuery_Client'][1]+" "
+ Clnt = exe['XQuery_Client'][1]+" "
for f in TSTs:
- RunIt(Clnt+f, "", ClntOut, ClntErr)
+ killed = RunIt(Clnt+f, "", ClntOut, ClntErr, TIMEOUT)
+ if killed:
+ break
elif CALL in ["x100", "x100Xs"]:
TSTs = []
if CALL == "x100":
@@ -2286,9 +2323,11 @@
if test.match(f):
TSTs.append(f)
- Clnt = MTO+str(TOT)+" "+exe['X100_Client'][1]
+ Clnt = exe['X100_Client'][1]
for f in TSTs:
- RunIt(Clnt, open(f), ClntOut, ClntErr)
+ killed = RunIt(Clnt, open(f), ClntOut, ClntErr, TIMEOUT)
+ if killed:
+ break
else:
for fp in ClntOut,ClntErr:
fp.write('\n\n! Server not ready; skipping attempt to start
client!\n\n')
@@ -2335,7 +2374,17 @@
if MkillUsersAT:
ATJOB2 = StartAt(MkillUsersAT)
- CollectIt(pSrvr.stdout, SrvrOut)
+ pSrvr.killed = False
+ t = threading.Timer(TIMEOUT, killProc, args = [pSrvr, SrvrErr, Srvr])
+ try:
+ t.start()
+ CollectIt(pSrvr.stdout, SrvrOut)
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(pSrvr, SrvrErr, Srvr)
+ raise
+ killed = killed or pSrvr.killed
if MkillUsersAT:
StopAt(ATJOB2,ME)
@@ -2378,15 +2427,26 @@
if TestErr is not None:
TestErr.write(Prompt('Done.'))
TestErr.close()
+ return killed
### DoIt(env, SERVER, CALL, TST, EXT, PRELUDE, TestOut, TestErr, STIMEOUT,
CTIMEOUT, TIMEOUT, MkillUsersAT, ME, MAPIsockets, XRPCsockets) #
def Check(command, input) :
- proc = subprocess.Popen(command+" || echo ! Exit 1", shell = True, stdin =
subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE,
universal_newlines = True)
- qOut, qErr = proc.communicate(input = input)
+ proc = subprocess.Popen(command, shell = True, stdin = subprocess.PIPE,
stdout = subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True)
+ t = threading.Timer(float(par['TIMEOUT']), killProc, args = [proc])
+ try:
+ t.start()
+ qOut, qErr = proc.communicate(input = input)
+ t.cancel()
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc)
+ raise
qOut = qOut.split('\n')
qErr = qErr.split('\n')
+ if proc.returncode:
+ qOut.append('! Exit 1')
test = re.compile( r"^!WARNING: BATpropcheck: "
"|"
r"^!WARNING: monet_checkbat: "
"|"
r"^!WARNING: GDKlockHome: ignoring empty or invalid
.gdk_lock." "|"
@@ -2871,12 +2931,9 @@
if THISFILE == "Mtest.py":
par['IGNORE'] = opts.get('ignore', dftIGNORE)
par['CONTEXT'] = '-C%d' % int(opts.get('context', 1))
- a = int(opts.get('accuracy', (os.name != 'nt' and 1) or 0))
+ a = int(opts.get('accuracy', 1))
if a not in (-1,0,1,2):
ErrExit('Accuracy for diff (-A) must be one of: 0=lines, 1=words,
2=chars !')
- if os.name == "nt" and a != 0:
- Warn("Currently only '-A0` is supported on WindowsNT!")
- a = 0
par['ACCURACY'] = a
par['TIMEOUT'] = int(opts.get('timeout', 60))
a = opts.get('debug')
@@ -3288,7 +3345,7 @@
QUIT = 'quit();\n'
if par['PACKAGE'] not in ('monetdb', 'monetdb-clients'):
- if Check('%s%s --dbname=%s' % (env['exe']['Mtimeout'][1],
env['exe']['Mserver'][1], TSTPREF), QUIT):
+ if Check('%s --dbname=%s' % (env['exe']['Mserver'][1], TSTPREF),
QUIT):
sys.exit(1)
if GetBitsAndOIDsAndModsAndStatic(env):
sys.exit(1)
Index: subprocess26.py
===================================================================
RCS file: /cvsroot/monetdb/MonetDB/src/testing/subprocess26.py,v
retrieving revision 1.2
retrieving revision 1.2.6.1
diff -u -d -r1.2 -r1.2.6.1
--- subprocess26.py 2 Apr 2007 20:40:17 -0000 1.2
+++ subprocess26.py 22 Oct 2007 13:25:08 -0000 1.2.6.1
@@ -1007,6 +1007,13 @@
if self.pid == 0:
# Child
try:
+ os.setpgrp()
+ except AttributeError:
+ try:
+ os.setpgid(0, 0)
+ except AttributeError:
+ pass
+ try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
_______________________________________________
Monetdb-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
