Changeset: 3d0c0cfe3ab8 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/3d0c0cfe3ab8
Modified Files:
testing/Mtest.py.in
Branch: Aug2024
Log Message:
Fixes for timeout handling.
- On the Mac, try to print a stack trace using lldb;
- When running in SingleServer mode, do not kill mserver5 but instead
stop all running sessions;
- Fix a race condition where the timeout handler, which runs in a
separate thread, is doing its thing and in the mean time the next test
is already started.
diffs (truncated from 318 to 300 lines):
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -1357,7 +1357,7 @@ def PerformDir(env, testdir, testlist, t
else:
vaultopt = []
if not oneserver:
- pSrvr = ServerClass(splitcommand(env['exe']['mserver5'][1]
+['--dbpath=%s' % LogDBdir] + vaultopt + mserver5_opts), open(os.devnull, 'w'),
open(os.devnull, 'w'), par['TIMEOUT'], os.path.join(LogDBdir, '.started'),
dbg=env.get('DBG'))
+ pSrvr = ServerClass(splitcommand(env['exe']['mserver5'][1]
+[f'--dbpath={LogDBdir}'] + vaultopt + mserver5_opts), open(os.devnull, 'w'),
open(os.devnull, 'w'), par['TIMEOUT'], os.path.join(LogDBdir, '.started'),
TSTDB, dbg=env.get('DBG'))
pSrvr.LaunchIt()
pSrvr.terminate()
if not os.path.exists(TSTTRGDIR):
@@ -1405,6 +1405,7 @@ def PerformDir(env, testdir, testlist, t
openutf8(os.path.join(TSTTRGDIR,
'SingleServer.err'), 'a'),
0,
pollfile,
+ TSTDB,
inmem=inmem,
dbg=env.get('DBG'))
os.chdir(TSTTRGDIR)
@@ -1419,9 +1420,14 @@ def PerformDir(env, testdir, testlist, t
hostname=HOST,
port=int(pSrvr.port),
database=TSTDB,
- autocommit=True)
+ autocommit=True,
+ connect_timeout=1.0)
except KeyboardInterrupt:
raise
+ except ConnectionRefusedError:
+ print('\nConnection refused.\n')
+ FdOut = FdErr = F_FAIL
+ break
except:
pass
else:
@@ -2567,16 +2573,7 @@ def reallyKill(proc):
# not called on Windows
killchildren(proc.pid, getkids())
-def killProc(proc, outfile = None, cmd = None):
- if type(cmd) is type([]):
- cmd = ' '.join(cmd)
- if procdebug:
- print('timeout for process %d (%s)' % (proc.pid, cmd))
- if outfile is not None and cmd is not None:
- try:
- outfile.write('\n!Mtimeout: Timeout: %s\n' % cmd)
- except (ValueError, IOError):
- print('cannot write timeout message ' + cmd)
+def stacktrace(proc, outfile):
if os.name == "nt":
sym = ''
if os.path.exists(r'c:\Program Files\Debugging Tools for Windows
(x64)\cdb.exe'):
@@ -2591,14 +2588,22 @@ def killProc(proc, outfile = None, cmd =
cdb = None
if cdb:
with process.Popen([cdb, '-pv', '-p', str(proc.pid),
- '-y',
'%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines',
'-c', '~*kP;!locks;q'],
+ '-y',
f'{sym}cache*;srv*http://msdl.microsoft.com/download/symbols', '-lines', '-c',
'~*kP;!locks;q'],
stdout=process.PIPE, text=True) as p:
out, err = p.communicate()
else:
out = ''
+ elif platform.system() == 'Darwin':
+ try:
+ with process.Popen(['lldb', '--attach-pid', str(proc.pid),
'--batch', '--one-line', 'bt all'], stdout=process.PIPE, text=True) as p:
+ out, err = p.communicate()
+ except KeyboardInterrupt:
+ raise
+ except:
+ out = ''
else:
try:
- with process.Popen(['gdb', '-p', str(proc.pid), '-batch', '-ex',
'thread apply all bt full', '-ex', 'call dump_threads()'], stdout=process.PIPE,
+ with process.Popen(['gdb', '-p', str(proc.pid), '-batch', '-ex',
'thread apply all bt full'], stdout=process.PIPE,
text=True) as p:
try:
# gdb sometimes hangs when trying to get the stack
@@ -2616,15 +2621,32 @@ def killProc(proc, outfile = None, cmd =
out = ''
if outfile is not None and out:
try:
- outfile.write('\n%s\n' % out)
+ outfile.write(f'\n{out}\n')
except (ValueError, IOError):
print('cannot write stack trace')
print(out)
+
+def killProc(proc, outfile=None, cmd=None, psrvr=None):
+ if type(cmd) is type([]):
+ cmd = ' '.join(cmd)
+ if procdebug:
+ print('timeout for process %d (%s)' % (proc.pid, cmd))
+ if outfile is not None and cmd is not None:
+ try:
+ outfile.write('\n!Mtimeout: Timeout: %s\n' % cmd)
+ except (ValueError, IOError):
+ print('cannot write timeout message ' + cmd)
+ stacktrace(proc, outfile)
proc.killed = True
if proc.onechild:
- if procdebug:
- print('killProc: calling proc.kill() on PID %d' % proc.pid)
- proc.kill()
+ if psrvr is not None and psrvr.proc is proc:
+ if procdebug:
+ print(f'killProc: calling psrvr.terminate() on PID {proc.pid}')
+ psrvr.terminate()
+ else:
+ if procdebug:
+ print(f'killProc: calling proc.kill() on PID {proc.pid}')
+ proc.kill()
elif os.name == 'nt':
if procdebug:
print('killProc: starting process "taskkill" "/F" "/T" "/PID"
"%s"\n' % str(proc.pid))
@@ -2639,7 +2661,7 @@ def killProc(proc, outfile = None, cmd =
killchildren(proc.pid)
class ServerClass:
- def __init__(self, cmd, TestOut, TestErr, TimeOut, pollfile, inmem=False,
dbg=None):
+ def __init__(self, cmd, TestOut, TestErr, TimeOut, pollfile, dbname,
inmem=False, dbg=None):
self.proc = None
self.timer = None
self.outfile = TestOut
@@ -2651,13 +2673,16 @@ class ServerClass:
self.pollfile = pollfile
self.inmem = inmem
self.dbg = dbg
+ self.dbname = dbname
+ self.running = None
+ self.lock = threading.Lock()
def poll(self):
return self.proc.poll()
def terminate(self):
self.timer.cancel()
- t = Timer(60, killProc, args = [self.proc, self.errfile, self.cmd])
+ t = Timer(60, killProc, args=[self.proc, self.errfile, self.cmd, self])
t.start()
if os.name == 'nt':
self.proc.send_signal(signal.CTRL_BREAK_EVENT)
@@ -2671,11 +2696,53 @@ class ServerClass:
self.outfile.close()
self.errfile.close()
+ def start(self, timeout):
+ self.running = Timer(timeout, self.stopsessions, [])
+ self.running.start()
+
+ def stop(self):
+ if self.lock.acquire():
+ if self.running:
+ self.running.cancel()
+ self.running = None
+ self.lock.release()
+
def sendusr1(self):
if os.name != 'nt':
+ if procdebug:
+ print(f'sendusr1: sending USR1 signal to {self.proc.pid}')
self.proc.send_signal(signal.SIGUSR1)
time.sleep(1)
+ def stacktrace(self):
+ stacktrace(self.proc, self.outfile)
+ self.sendusr1()
+
+ def stopsessions(self):
+ if self.lock.acquire(blocking=False):
+ if self.running is not None:
+ self.stacktrace()
+ try:
+ dbh = pymonetdb.connect(username='monetdb',
+ password='monetdb',
+ hostname=HOST,
+ port=int(self.port),
+ database=self.dbname,
+ connect_timeout=1.0)
+ crs = dbh.cursor()
+ crs.execute('select sessionid from sys.sessions() where
sessionid <> sys.current_sessionid()')
+ ids = crs.fetchall()
+ for x in ids:
+ if procdebug:
+ print(f'stopping session {x[0]}')
+ crs.execute(f'call sys.stopsession({x[0]})')
+ if procdebug and not ids:
+ print('no sessions to stop')
+ except:
+ pass
+ self.running = None
+ self.lock.release()
+
def LaunchIt(self):
global setpgrp
@@ -2715,11 +2782,10 @@ class ServerClass:
proc.stderr = process._BufferedPipe(proc.stderr)
proc.killed = False
proc.onechild = True
- t = Timer(self.timeout, killProc, args = [proc, self.errfile,
self.cmd])
+ self.timer = Timer(self.timeout, killProc, args=[proc, self.errfile,
self.cmd, self])
if self.timeout > 0:
- t.start()
+ self.timer.start()
self.proc = proc
- self.timer = t
port = None
self.port = None
@@ -2729,7 +2795,7 @@ class ServerClass:
if proc.returncode is not None:
# exited
proc.wait()
- t.cancel()
+ self.timer.cancel()
return
if os.path.exists(self.pollfile):
break
@@ -2756,14 +2822,14 @@ class ServerClass:
else:
proc.terminate()
proc.wait()
- t.cancel()
+ self.timer.cancel()
return
self.started = True
self.port = port
### LaunchIt(cmd, TestIn, TestOut, TestErr, TimeOut, pollfile, port) #
-def RunIt(cmd, onechild, TestIn, TestOut, TestErr, TimeOut) :
+def RunIt(cmd, onechild, TestIn, TestOut, TestErr, TimeOut, pSrvr) :
global setpgrp
if type(TestIn) is type(''):
TestInput = TestIn
@@ -2779,7 +2845,7 @@ def RunIt(cmd, onechild, TestIn, TestOut
stderr=TestErr, text=True) as proc:
proc.killed = False
proc.onechild = onechild
- t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd])
+ t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd, pSrvr])
try:
t.start()
# since both stdout and stderr are redirected to files,
@@ -2916,7 +2982,7 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te
Srvr.extend(openutf8(TST + '.options5').read().split())
Srvr.extend(mserver5_opts)
Srvr.extend(options)
- pSrvr = ServerClass(Srvr, SrvrOut, SrvrErr, TIMEOUT,
os.path.join(dbpath, '.started'), dbg=env.get('DBG'))
+ pSrvr = ServerClass(Srvr, SrvrOut, SrvrErr, TIMEOUT,
os.path.join(dbpath, '.started'), TSTDB, dbg=env.get('DBG'))
pSrvr.LaunchIt()
if pSrvr.port is None:
print('\nFailed to start server.\n')
@@ -2924,8 +2990,9 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te
os.environ['MAPIPORT'] = env['MAPIPORT'] = pSrvr.port
SetExecEnv(exe,pSrvr.port,verbosity > 1)
else:
- PSRVR.timer.settimeout(TIMEOUT)
- PSRVR.timer.start()
+ PSRVR.start(TIMEOUT)
+ # PSRVR.timer.settimeout(TIMEOUT)
+ # PSRVR.timer.start()
else:
ClntOut = openutf8(TestOutFile, 'a')
ClntErr = openutf8(TestErrFile, 'a')
@@ -2933,7 +3000,7 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te
if SERVER not in ["MAL", "SQL"] or pSrvr.started:
if CALL == "other":
cmd = [os.path.join(".", TST + EXT), TST]
- returncode = RunIt(cmd, False, "", ClntOut, ClntErr, TIMEOUT)
+ returncode = RunIt(cmd, False, "", ClntOut, ClntErr, TIMEOUT,
pSrvr)
elif CALL == "python":
# do clean up between tests if no dependent tests
# borrow clean up function from sqllogictest
@@ -2961,7 +3028,7 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te
except:
pass
cmd = [sys.executable, TST + EXT, TST]
- returncode = RunIt(cmd, False, "", ClntOut, ClntErr, TIMEOUT)
+ returncode = RunIt(cmd, False, "", ClntOut, ClntErr, TIMEOUT,
pSrvr)
elif CALL == "sqltest" or CALL == "maltest":
issqllogictest = True
import MonetDBtesting.sqllogictest as sqllogictest
@@ -3027,19 +3094,19 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te
if passwd:
Clnt.append('-P%s' % passwd)
for f in TSTs:
- returncode = RunIt(Clnt, True, openutf8(f), ClntOut,
ClntErr, TIMEOUT)
+ returncode = RunIt(Clnt, True, openutf8(f), ClntOut,
ClntErr, TIMEOUT, pSrvr)
if returncode:
break
elif CALL == "R":
Clnt = splitcommand(exe['R_Client'][1].replace('${PORT}',
env['MAPIPORT']))
- RunIt(Clnt, False, openutf8(TST+EXT), ClntOut, ClntErr,
TIMEOUT)
+ RunIt(Clnt, False, openutf8(TST+EXT), ClntOut, ClntErr,
TIMEOUT, pSrvr)
elif CALL == "ruby":
Clnt = splitcommand(exe['ruby_client'][1].replace('${PORT}',
env['MAPIPORT'])) + [TST + EXT]
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
