Fix the LocalExecutor
http://reviews.llvm.org/D7380
Files:
test/libcxx/test/config.py
test/libcxx/test/format.py
test/libcxx/test/remote.py
test/lit.site.cfg.in
test/remoteRun.py
EMAIL PREFERENCES
http://reviews.llvm.org/settings/panel/emailpreferences/
Index: test/libcxx/test/config.py
===================================================================
--- test/libcxx/test/config.py
+++ test/libcxx/test/config.py
@@ -11,7 +11,7 @@
from libcxx.test.format import LibcxxTestFormat
from libcxx.compiler import CXXCompiler
-
+from libcxx.test.remote import *
def loadSiteConfig(lit_config, config, param_name, env_name):
# We haven't loaded the site specific configuration (the user is
@@ -80,6 +80,7 @@
"parameter '{}' should be true or false".format(name))
def configure(self):
+ self.configure_target_executor()
self.configure_cxx()
self.configure_triple()
self.configure_src_root()
@@ -113,8 +114,15 @@
self.cxx,
self.use_clang_verify,
self.execute_external,
+ self.target_executor,
exec_env=self.env)
+ def configure_target_executor(self):
+ exec_str = self.get_lit_conf('target_executor', None)
+ if exec_str:
+ self.target_executor = eval(exec_str)
+ self.lit_config.note("inferred target_executor as: %r" % exec_str)
+
def configure_cxx(self):
# Gather various compiler parameters.
cxx = self.get_lit_conf('cxx_under_test')
@@ -257,6 +265,14 @@
self.config.available_features.add(
'with_system_cxx_lib=%s' % self.config.target_triple)
+ # Insert the platform name into the available features as a lower case.
+ # Strip the '2' from linux2.
+ if sys.platform.startswith('linux'):
+ platform_name = 'linux'
+ else:
+ platform_name = sys.platform
+ self.config.available_features.add(platform_name.lower())
+
# Some linux distributions have different locale data than others.
# Insert the distributions name and name-version into the available
# features to allow tests to XFAIL on them.
Index: test/libcxx/test/format.py
===================================================================
--- test/libcxx/test/format.py
+++ test/libcxx/test/format.py
@@ -20,10 +20,12 @@
FOO.sh.cpp - A test that uses LIT's ShTest format.
"""
- def __init__(self, cxx, use_verify_for_fail, execute_external, exec_env):
+ def __init__(self, cxx, use_verify_for_fail, execute_external,
+ target_executor, exec_env):
self.cxx = cxx
self.use_verify_for_fail = use_verify_for_fail
self.execute_external = execute_external
+ self.target_executor = target_executor
self.exec_env = dict(exec_env)
# TODO: Move this into lit's FileBasedTest
@@ -73,21 +75,27 @@
# Dispatch the test based on its suffix.
if is_sh_test:
+ if self.target_executor:
+ # We can't run ShTest tests with a target_executor yet.
+ # For now, bail on trying to run them
+ return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
return lit.TestRunner._runShTest(test, lit_config,
self.execute_external, script,
tmpBase, execDir)
elif is_fail_test:
return self._evaluate_fail_test(test)
elif is_pass_test:
- return self._evaluate_pass_test(test, tmpBase, execDir, lit_config)
+ return self._evaluate_pass_test(test, tmpBase, execDir, lit_config,
+ self.target_executor)
else:
# No other test type is supported
assert False
def _clean(self, exec_path): # pylint: disable=no-self-use
libcxx.util.cleanFile(exec_path)
- def _evaluate_pass_test(self, test, tmpBase, execDir, lit_config):
+ def _evaluate_pass_test(self, test, tmpBase, execDir, lit_config,
+ target_executor):
source_path = test.getSourcePath()
exec_path = tmpBase + '.exe'
object_path = tmpBase + '.o'
@@ -110,9 +118,23 @@
cmd += ['%s=%s' % (k, v) for k, v in self.exec_env.items()]
if lit_config.useValgrind:
cmd = lit_config.valgrindArgs + cmd
- cmd += [exec_path]
- out, err, rc = lit.util.executeCommand(
- cmd, cwd=os.path.dirname(source_path))
+ if target_executor:
+ target_exec_path = target_executor.remote_from_local_file(exec_path)
+ local_cwd = os.path.dirname(source_path)
+ target_cwd = target_executor.remote_from_local_dir(local_cwd)
+ cmd += [target_exec_path]
+ try:
+ target_executor.copy_in([exec_path], [target_exec_path])
+ out, err, rc = target_executor.run(cmd, target_cwd)
+ except:
+ raise
+ finally:
+ target_executor.delete_remote(target_exec_path)
+ pass
+ else:
+ cmd += [exec_path]
+ out, err, rc = lit.util.executeCommand(
+ cmd, cwd=os.path.dirname(source_path))
if rc != 0:
report = libcxx.util.makeReport(cmd, out, err, rc)
report = "Compiled With: %s\n%s" % (compile_cmd, report)
@@ -123,7 +145,9 @@
# Note that cleanup of exec_file happens in `_clean()`. If you
# override this, cleanup is your reponsibility.
libcxx.util.cleanFile(object_path)
- self._clean(exec_path)
+ # If there's a target_executor, let it do its own exe cleanup
+ if not target_executor:
+ self._clean(exec_path)
def _evaluate_fail_test(self, test):
source_path = test.getSourcePath()
Index: test/libcxx/test/remote.py
===================================================================
--- test/libcxx/test/remote.py
+++ test/libcxx/test/remote.py
@@ -0,0 +1,397 @@
+import os
+import shutil
+
+# FIXME:
+# Shamelessly stolen from lit/util.py, so we don't
+# have to deal with a lit import:
+import signal
+import platform
+import subprocess
+def to_bytes(str):
+ # Encode to UTF-8 to get binary data.
+ return str.encode('utf-8')
+
+def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return to_bytes(bytes)
+
+def convert_string(bytes):
+ try:
+ return to_string(bytes.decode('utf-8'))
+ except UnicodeError:
+ return str(bytes)
+
+# Close extra file handles on UNIX (on Windows this cannot be done while
+# also redirecting input).
+kUseCloseFDs = not (platform.system() == 'Windows')
+def executeCommand(command, cwd=None, env=None):
+ p = subprocess.Popen(command, cwd=cwd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env, close_fds=kUseCloseFDs)
+ out,err = p.communicate()
+ exitCode = p.wait()
+
+ # Detect Ctrl-C in subprocess.
+ if exitCode == -signal.SIGINT:
+ raise KeyboardInterrupt
+
+ # Ensure the resulting output is always of string type.
+ out = convert_string(out)
+ err = convert_string(err)
+
+ return out, err, exitCode
+
+
+
+class Executor(object):
+ def __init__(self):
+ pass
+
+ def remote_temp_dir(self):
+ """Get a temporary directory on the remote host"""
+ pass
+
+ def remote_temp_file(self, suffix):
+ """Get a temporary filename on the remote host"""
+ pass
+
+ def remote_from_local_dir(self, local_dir):
+ """Translate a local dirname into a remote dirname"""
+ pass
+
+ def remote_from_local_file(self, local_file):
+ """Translate a local filename into a remote filename"""
+ pass
+
+ def copy_in(self, local_srcs, remote_dsts, delete=False):
+ """Copy test inputs & executables in to the test runner
+ Args:
+ local_srcs (list): List of Host file paths to copy in to the runner
+ remote_dsts (list): Corresponding list of Target file locations
+ delete (bool): Whether or not to delete the src after the copy
+ Returns:
+ void
+ """
+ pass
+
+ def copy_out(self, remote_srcs, local_dsts, delete=False):
+ """Copy test outputs back from the test runner
+ Args:
+ remote_srcs (list): List of Target file paths to copy back from the runner
+ local_dsts (list): Corresponding list of Host file locations
+ delete (bool): Whether or not to delete the src after the copy
+ Returns:
+ void
+ """
+ pass
+
+ def delete_remote(self, remote, ignore_error=True):
+ """Delete remote files
+ Args:
+ remote (path): Remote file to delete
+ ignore_error (bool): Whether or not to ignore errors on delete
+ Returns:
+ void
+ """
+ pass
+
+ def run(self, command, work_dir='.'):
+ """Execute a particular command
+ Args:
+ command (string): The command to execute
+ work_dir (string): The working directory to execute the test from
+ Returns:
+ out, err, exitCode
+ """
+ pass
+
+ def clean_temps(self):
+ """Delete all temporary files created by this executor"""
+ pass
+
+class LocalExecutor(Executor):
+ def __init__(self):
+ Executor.__init__(self)
+
+ def remote_temp_dir(self):
+ return os.getcwd()
+
+ def remote_temp_file(self, suffix):
+ tmp_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
+ tmp_path = tmp_file.name
+ tmp_file.close()
+ return tmp_path
+
+ def delete_remote(self, remote, ignore_error=True):
+ try:
+ os.remove(remote)
+ except:
+ if ignore_error:
+ pass
+ else:
+ raise
+
+ def remote_from_local_dir(self, local_dir):
+ # Identity function, because LocalExecutor executes locally
+ return local_dir
+
+ def remote_from_local_file(self, local_file):
+ # Identity function, because LocalExecutor executes locally
+ return local_file
+
+ def copy_in(self, local_srcs, remote_dsts, delete=False):
+ self.__copyAll(local_srcs, remote_dsts, delete)
+
+ def copy_out(self, remote_srcs, local_dsts, delete=False):
+ self.__copyAll(remote_srcs, local_dsts, delete)
+
+ def run(self, command, work_dir='.'):
+ return executeCommand(command, cwd=work_dir)
+
+ def __copyAll(self, srcs, dsts, delete=False):
+ for src, dst in zip(srcs, dsts):
+ if src is not dst:
+ if delete:
+ move(src, dst)
+ else:
+ shutil.copy(src, dst)
+ elif delete:
+ assert false, "Can't delete '%s' because src==dst" % src
+
+class PrefixExecutor(Executor):
+ """Prefix an executor with some other command wrapper
+ Most useful for setting ulimits on commands, or running an emulator like
+ qemu and valgrind.
+ """
+ def __init__(self, commandPrefix, chain):
+ Executor.__init__(self)
+
+ self.commandPrefix = commandPrefix
+ self.chain = chain
+
+ def remote_from_local_dir(self, local_dir):
+ return local_dir
+
+ def remote_from_local_file(self, local_file):
+ return local_file
+
+ def copy_in(self, local_srcs, remote_dsts, delete=False):
+ self.chain.copy_in(local_srcs, remote_dsts, delete)
+
+ def copy_out(self, remote_srcs, local_dsts, delete=False):
+ self.chain.copy_out(remote_srcs, local_dsts, delete)
+
+ def delete_remote(self, remote, ignore_error=True):
+ self.chain.delete_remote(remote, ignore_error)
+
+ def run(self, command, work_dir='.'):
+ return self.chain.run(self.commandPrefix + command, work_dir)
+
+class ValgrindExecutor(PrefixExecutor):
+ pass
+
+class QEMUExecutor(PrefixExecutor):
+ """Execute another action under qemu"""
+ def __init__(self, qemuCommand, qemuArgs, chain):
+ PrefixExecutor.__init__(self, prefix, chain)
+ self.prefix = qemuCommand + ' ' + qemuArgs
+
+class TimeoutExecutor(PrefixExecutor):
+ """Execute another action under a timeout"""
+ def __init__(self, duration, chain):
+ PrefixExecutor.__init__(self, 'timeout ' + duration, chain)
+
+class SSHExecutor(Executor):
+ def __init__(self, username, host):
+ Executor.__init__(self)
+
+ self.username = username
+ self.host = host
+ self.scpCommand = 'scp'
+ self.sshCommand = 'ssh'
+ self.local_executor = LocalExecutor()
+
+ # Maps from local->remote to make sure we consistently return
+ # the same name for a given local version of the same file/dir
+ self.filemap = {}
+ self.dirmap = {}
+
+ # For temp cleanup
+ self.remote_temp_files = []
+ self.remote_temp_dirs = []
+
+ # TODO(jroelofs): switch this on some -super-verbose-debug config flag
+ if False:
+ self.local_executor = DebugExecutor("ssh_local", LocalExecutor(),
+ before=True, after=True)
+
+ def remote_temp_dir(self):
+ # TODO
+ return os.getcwd()
+
+ def remote_temp_file(self, suffix):
+ # Not sure how to do suffix on osx yet
+ cmd = "mktemp -q /tmp/libcxx.XXXXXXXXXX"
+ res = self.__execute_command_remote([cmd])
+ temp_path, err, exitCode = res
+ temp_path = temp_path.rstrip()
+ if exitCode != 0:
+ raise ValueError(err)
+ self.remote_temp_files.append(temp_path)
+ return temp_path
+
+ def remote_from_local_dir(self, local_dir):
+ if not self.dirmap.has_key(local_dir):
+ remote_dir = self.remote_temp_dir()
+ self.dirmap[local_dir] = remote_dir
+ return remote_dir
+ else:
+ return self.dirmap[local_dir]
+
+ def remote_from_local_file(self, local_file):
+ if not self.filemap.has_key(local_file):
+ remote_file = self.remote_temp_file(".some_file")
+ self.filemap[local_file] = remote_file
+ return remote_file
+ else:
+ return self.filemap[local_file]
+
+ def copy_in(self, local_srcs, remote_dsts, delete=False):
+ scp = self.scpCommand
+ remote = self.username + '@' + self.host
+
+ # This could be wrapped up in a tar->scp->untar for performance
+ # if there are lots of files to be copied/moved
+ for src, dst in zip(local_srcs, remote_dsts):
+ cmd = [scp, '-p', src, remote + ':' + dst]
+ self.local_executor.run(cmd)
+ if delete:
+ for src in local_srcs:
+ os.remove(src)
+
+ def copy_out(self, remote_srcs, local_dsts, delete=False):
+ scp = self.scpCommand
+ remote = self.username + '@' + self.host
+
+ # This could be wrapped up in a tar->scp->untar for performance
+ # if there are lots of files to be copied/moved
+ for src, dst in zip(srcs, dsts):
+ cmd = [scp, '-p', remote + ':' + src, dst]
+ self.local_executor.run(cmd)
+ if delete:
+ for src in remote_srcs:
+ self.delete_remote(src)
+
+ def delete_remote(self, remote, ignore_error=True):
+ try:
+ self.__execute_command_remote(['rm', '-rf', remote])
+ except:
+ if ignore_error:
+ pass
+ else:
+ raise
+
+ def run(self, command, remote_work_dir='.'):
+ return self.__execute_command_remote(command, remote_work_dir)
+
+ def __execute_command_remote(self, command, remote_work_dir=''):
+ remote = self.username + '@' + self.host
+ cmd = [self.sshCommand]
+ cmd += ['-oBatchMode=yes']
+ cmd += [remote]
+ remote_cmd = ' '.join(command)
+ if remote_work_dir is not '.' and remote_work_dir is not '':
+ remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd
+ cmd += [remote_cmd]
+ return self.local_executor.run(cmd)
+
+ def clean_temps(self):
+ for remote_file in self.remote_temp_files:
+ self.delete_remote(remote_file)
+
+ for remote_dir in self.remote_temp_dirs:
+ self.delete_remote(remote_dir)
+
+class DebugExecutor(Executor):
+ def __init__(self, label, chain, before=True, after=True):
+ Executor.__init__(self)
+ self.label = label
+ self.chain = chain
+ self.before = before
+ self.after = after
+
+ def remote_temp_dir(self, suffix):
+ self.__log_before("remote_temp_dir('%s')" % (suffix))
+ res = self.chain.remote_temp_dir(suffix)
+ self.__log_after("remote_temp_dir('%s') = %s" % (suffix, res))
+ return res
+
+ def remote_temp_file(self, suffix):
+ self.__log_before("remote_temp_file('%s')" % (suffix))
+ res = self.chain.remote_temp_file(suffix)
+ self.__log_after("remote_temp_file('%s') = %s" % (suffix, res))
+ return res
+
+ def delete_remote(self, remote, ignore_error=True):
+ try:
+ self.__log_before("delete_remote('%s', %s)" % (remote, ignore_error))
+ res = self.chain.delete_remote(remote, ignore_error)
+ self.__log_after("delete_remote('%s', %s)" % (remote, ignore_error))
+ except:
+ self.__log_after("delete_remote('%s', %s) raised" % (remote, ignore_error))
+ if ignore_error:
+ pass
+ else:
+ raise
+
+ def remote_from_local_dir(self, local):
+ self.__log_before("remote_from_local_dir('%s')" % (local))
+ res = self.chain.remote_from_local_dir(local)
+ self.__log_after("remote_from_local_dir('%s') = %s" % (local, res))
+
+ def remote_from_local_file(self, local):
+ self.__log_before("remote_from_local_file('%s')" % (local))
+ res = self.chain.remote_from_local_file(local)
+ self.__log_after("remote_from_local_file('%s') = %s" % (local, res))
+ return res
+
+ def copy_in(self, local_srcs, remote_dsts, delete=False):
+ self.__log_before("copy_in(%s, %s, %s)" % (local_srcs, remote_dsts, delete))
+ res = self.chain.copy_in(local_srcs, remote_dsts, delete)
+ self.__log_after("copy_in(%s, %s, %s) = %s" % (local_srcs, remote_dsts, delete, res))
+ return res
+
+ def copy_out(self, remote_srcs, local_dsts, delete=False):
+ self.__log_before("copy_out(%s, %s, %s)" % (remote_srcs, local_dsts, delete))
+ res = self.chain.copy_out(remote_srcs, local_dsts, delete)
+ self.__log_after("copy_out(%s, %s, %s) = %s" % (remote_srcs, local_dsts, delete, res))
+ return res
+
+ def run(self, command, work_dir='.'):
+ try:
+ self.__log_before("run(%s, '%s')" % (command, work_dir))
+ res = self.chain.run(command, work_dir)
+ self.__log_after("run(%s, '%s') = %s" % (command, work_dir, res))
+ return res
+ except:
+ self.__log_after("run(%s, '%s') raised" % (command, work_dir))
+ raise
+
+ def clean_temps(self):
+ self.__log_before("clean_temps()")
+ res = self.chain.clean_temps()
+ self.__log_after("clean_temps() = %s" % (res))
+ return res
+
+ def __log_before(self, message):
+ if self.before:
+ print "BEFORE: %s.%s" % (self.label, message)
+
+ def __log_after(self, message):
+ if self.after:
+ print "AFTER: %s.%s" % (self.label, message)
+
+
Index: test/lit.site.cfg.in
===================================================================
--- test/lit.site.cfg.in
+++ test/lit.site.cfg.in
@@ -17,6 +17,9 @@
config.target_triple = "@LIBCXX_TARGET_TRIPLE@"
config.sysroot = "@LIBCXX_SYSROOT@"
config.gcc_toolchain = "@LIBCXX_GCC_TOOLCHAIN@"
+#config.target_executor = "SSHExecutor('jroelofs', 'galois.local')"
+#config.target_executor = "SSHExecutor('jroelofs', '127.0.0.1')"
+config.target_executor = "LocalExecutor()"
# Let the main config do the real work.
lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")
Index: test/remoteRun.py
===================================================================
--- test/remoteRun.py
+++ test/remoteRun.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import sys
+
+from libcxx.test.remote import *
+
+def main():
+ # TODO(jroelofs): what's the best way of making this configurable?
+ executor = SSHExecutor('jroelofs', '127.0.0.1')
+
+ remote_wd = executor.remote_from_local_dir(os.getcwd())
+
+ args = sys.argv[1:]
+ out, err, rc = executor.run(args, remote_wd)
+
+ sys.stderr.write(err)
+ sys.stdout.write(out)
+
+ executor.clean_temps()
+
+ sys.exit(rc)
+
+if __name__ == "__main__":
+ main()
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
