Add support for building and generating code coverage using lcov and genhtml.
To produce code coverage you must:
* configure with '-DLIBCXX_GENERATE_COVERAGE=ON'
* `make`
* `make check-libcxx`
* `make check-libcxx-coverage`
The resulting HTML can be found in
`<build-libcxx>/test/coverage/libcxx_coverage`
This also adds support for `UNSUPPORTED-XFAIL` and `REQUIRES-XFAIL` tags that
ensure compilation fail but report the test as UNSUPPORTED instead of XFAIL.
http://reviews.llvm.org/D5877
Files:
CMakeLists.txt
cmake/Modules/LibcxxCodeCoverage.cmake
lib/CMakeLists.txt
test/CMakeLists.txt
test/lit.cfg
test/lit.site.cfg.in
www/index.html
www/lit_usage.html
Index: CMakeLists.txt
===================================================================
--- CMakeLists.txt
+++ CMakeLists.txt
@@ -43,6 +43,7 @@
option(LIBCXX_ENABLE_CXX1Y "Enable -std=c++1y and use of c++1y language features if the compiler supports it." OFF)
option(LIBCXX_ENABLE_SHARED "Build libc++ as a shared library." ON)
option(LIBCXX_INSTALL_SUPPORT_HEADERS "Install libc++ support headers." ON)
+option(LIBCXX_GENERATE_COVERAGE "Generate code coverage results from the tests." OFF)
if (LIBCXX_BUILT_STANDALONE)
set(LLVM_USE_SANITIZER "" CACHE STRING
"Define the sanitizer used to build the library and tests")
@@ -231,6 +232,11 @@
string(REPLACE ";" " " LIBCXX_CXX_FEATURE_FLAGS "${LIBCXX_CXX_FEATURE_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBCXX_CXX_FEATURE_FLAGS}")
+if (LIBCXX_GENERATE_COVERAGE)
+ include(LibcxxCodeCoverage)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${LIBCXX_CXX_COVERAGE_FLAGS}")
+endif()
+
#===============================================================================
# Setup Source Code
#===============================================================================
Index: cmake/Modules/LibcxxCodeCoverage.cmake
===================================================================
--- /dev/null
+++ cmake/Modules/LibcxxCodeCoverage.cmake
@@ -0,0 +1,41 @@
+
+find_program(LIBCXX_GCOV gcov)
+if (NOT LIBCXX_GCOV)
+ message(FATAL_ERROR "Cannot find gcov...")
+endif()
+
+find_program(LIBCXX_LCOV lcov)
+find_program(LIBCXX_GENHTML genhtml)
+
+set(LIBCXX_CXX_COVERAGE_FLAGS "-g -O0 --coverage -fprofile-arcs -ftest-coverage")
+
+function(setup_lcov_test_target_coverage _coverage_dir)
+ if (NOT LIBCXX_LCOV)
+ message(FATAL_ERROR "Cannot find lcov...")
+ endif()
+ if (NOT LIBCXX_GENHTML)
+ message(FATAL_ERROR "Cannot find genhtml...")
+ endif()
+
+ file(MAKE_DIRECTORY ${_coverage_dir})
+
+ add_custom_command(TARGET cxx
+ PRE_BUILD
+ COMMAND ${LIBCXX_LCOV} --directory . --directory ${_SRC_BUILD_DIR} --zerocounters
+ WORKING_DIRECTORY ${_coverage_dir}
+ COMMENT "Reseting code coverage information.")
+
+ set(_SRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../lib/CMakeFiles/cxx.dir/__/src/)
+ set(_CHMOD_END "{} \\$<SEMICOLON>")
+ separate_arguments(_CHMOD_END)
+ add_custom_target(check-libcxx-coverage
+ COMMAND ${LIBCXX_LCOV} --directory . --directory ${_SRC_BUILD_DIR} --capture --output-file libcxx_coverage.info
+ COMMAND ${LIBCXX_LCOV} --remove libcxx_coverage.info 'test/*' 'test/*/*' '/usr/*' '/usr/include/*' --output-file libcxx_coverage.info
+ COMMAND ${LIBCXX_GENHTML} --demangle-cpp -o libcxx_coverage libcxx_coverage.info
+ COMMAND ${CMAKE_COMMAND} -E remove libcxx_coverage.info
+ COMMAND ${CMAKE_COMMAND} -E remove_directory ${_coverage_dir}/test
+ COMMAND find libcxx_coverage/ -type d -exec chmod 750 ${_CHMOD_END}
+ COMMAND find libcxx_coverage/ -type f -exec chmod 640 ${_CHMOD_END}
+ WORKING_DIRECTORY ${_coverage_dir}
+ COMMENT "Generating coverage results")
+endfunction()
Index: lib/CMakeLists.txt
===================================================================
--- lib/CMakeLists.txt
+++ lib/CMakeLists.txt
@@ -43,7 +43,7 @@
append_if(libraries LIBCXX_HAS_RT_LIB rt)
append_if(libraries LIBCXX_HAS_GCC_S_LIB gcc_s)
-#if LIBCXX_CXX_ABI_LIBRARY_PATH is defined we want to add it to the search path.
+# if LIBCXX_CXX_ABI_LIBRARY_PATH is defined we want to add it to the search path.
if (DEFINED LIBCXX_CXX_ABI_LIBRARY_PATH)
target_link_libraries(cxx "-L${LIBCXX_CXX_ABI_LIBRARY_PATH}")
endif()
@@ -51,7 +51,11 @@
# Setup flags.
append_if(compile_flags LIBCXX_HAS_FPIC_FLAG -fPIC)
-append_if(link_flags LIBCXX_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs)
+# FIXME: Currently -nodefaultlibs causes the tests not to link when using
+# code coverage due to the order of the libraries passed to the linker.
+if (NOT LIBCXX_GENERATE_COVERAGE)
+ append_if(link_flags LIBCXX_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs)
+endif()
if ( APPLE )
if ( CMAKE_OSX_DEPLOYMENT_TARGET STREQUAL "10.6" )
Index: test/CMakeLists.txt
===================================================================
--- test/CMakeLists.txt
+++ test/CMakeLists.txt
@@ -28,6 +28,10 @@
set(LIBCXX_BINARY_DIR ${CMAKE_BINARY_DIR})
set(PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE})
pythonize_bool(LIBCXX_ENABLE_SHARED)
+ pythonize_bool(LIBCXX_GENERATE_COVERAGE)
+ if (LIBCXX_GENERATE_COVERAGE)
+ set(LIBCXX_COVERAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/coverage")
+ endif()
set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
@@ -43,6 +47,11 @@
${CMAKE_CURRENT_BINARY_DIR}
DEPENDS cxx
COMMENT "Running libcxx tests")
+
+ if (LIBCXX_GENERATE_COVERAGE)
+ include(LibcxxCodeCoverage)
+ setup_lcov_test_target_coverage(${LIBCXX_COVERAGE_DIR})
+ endif()
else()
message(WARNING "Could not find Python, no check target will be available!")
endif()
Index: test/lit.cfg
===================================================================
--- test/lit.cfg
+++ test/lit.cfg
@@ -28,31 +28,53 @@
FOO.fail.cpp - Negative test case which is expected to fail compilation.
"""
- def __init__(self, cxx_under_test, use_verify_for_fail,
- cpp_flags, ld_flags, exec_env):
+ def __init__(self, cxx_under_test, cpp_flags, ld_flags, exec_env,
+ test_root,
+ use_verify_for_fail = False,
+ generate_coverage = False,
+ coverage_flags=[],
+ coverage_root = None,
+ use_scan_build = None,
+ scan_build_output = None,
+ scan_build_args = [],
+ use_clang_tidy = None,
+ clang_tidy_args = []):
self.cxx_under_test = cxx_under_test
self.use_verify_for_fail = use_verify_for_fail
self.cpp_flags = list(cpp_flags)
self.ld_flags = list(ld_flags)
self.exec_env = dict(exec_env)
-
- def execute_command(self, command, in_dir=None):
- kwargs = {
- 'stdin' :subprocess.PIPE,
- 'stdout':subprocess.PIPE,
- 'stderr':subprocess.PIPE,
- }
- if in_dir:
- kwargs['cwd'] = in_dir
- p = subprocess.Popen(command, **kwargs)
- out,err = p.communicate()
- exitCode = p.wait()
-
- # Detect Ctrl-C in subprocess.
- if exitCode == -signal.SIGINT:
- raise KeyboardInterrupt
-
- return out, err, exitCode
+ self.test_root = str(test_root)
+ self.generate_coverage = generate_coverage
+ self.coverage_flags = list(coverage_flags)
+ self.coverage_root = str(coverage_root)
+ self.use_scan_build = use_scan_build
+ self.scan_build_output = scan_build_output
+ self.scan_build_args = list(scan_build_args)
+ self.use_clang_tidy = use_clang_tidy
+ self.clang_tidy_args = list(clang_tidy_args)
+
+ def execute_command_and_report(self, command, cwd=None, env=None, compile_cmd=None):
+ exec_env = dict(os.environ)
+ if env:
+ exec_env.extend(env)
+ out,err,exitCode = lit.util.executeCommand(command, cwd=cwd, env=exec_env)
+ report = ''
+ if compile_cmd:
+ report += """Compiled with: %s\n""" % \
+ ' '.join(["'%s'" % a for a in compile_cmd])
+ report += """Command: %s\n""" % \
+ ' '.join(["'%s'" % a for a in command])
+ if env:
+ report += """Environment: %s\n""" % \
+ ' '.join(["'%s'='%s'" % (k,env[k]) for k in env])
+ report += """Exit Code: %d\n""" % exitCode
+ if out:
+ report += """Standard Output:\n--\n%s--""" % out
+ if err:
+ report += """Standard Error:\n--\n%s--""" % err
+ report += '\n\n'
+ return report, exitCode, out, err
def execute(self, test, lit_config):
while True:
@@ -63,22 +85,33 @@
raise
time.sleep(0.1)
+ def _handle_metadata_line(self, ln, tag, out):
+ if tag not in ln:
+ return False
+ items = ln[ln.index(tag) + len(tag):].split(',')
+ items = [s.strip() for s in items if s.strip()]
+ out.extend(items)
+ return True
+
def _execute(self, test, lit_config):
# Extract test metadata from the test file.
requires = []
unsupported = []
+ unsupported_xfail = []
+ requires_xfail = []
use_verify = False
with open(test.getSourcePath()) as f:
for ln in f:
- if 'XFAIL:' in ln:
- items = ln[ln.index('XFAIL:') + 6:].split(',')
- test.xfails.extend([s.strip() for s in items])
- elif 'REQUIRES:' in ln:
- items = ln[ln.index('REQUIRES:') + 9:].split(',')
- requires.extend([s.strip() for s in items])
- elif 'UNSUPPORTED:' in ln:
- items = ln[ln.index('UNSUPPORTED:') + 12:].split(',')
- unsupported.extend([s.strip() for s in items])
+ if self._handle_metadata_line(ln, 'REQUIRES-XFAIL:', requires_xfail):
+ pass
+ elif self._handle_metadata_line(ln, 'UNSUPPORTED-XFAIL:', unsupported_xfail):
+ pass
+ elif self._handle_metadata_line(ln, 'XFAIL:', test.xfails):
+ pass
+ elif self._handle_metadata_line(ln, 'REQUIRES:', requires):
+ pass
+ elif self._handle_metadata_line(ln, 'UNSUPPORTED:', unsupported):
+ pass
elif 'USE_VERIFY' in ln and self.use_verify_for_fail:
use_verify = True
elif not ln.strip().startswith("//") and ln.strip():
@@ -92,100 +125,144 @@
# introducing a dependency there. What we more ideally would like to do
# is lift the "requires" handling to be a core lit framework feature.
missing_required_features = [f for f in requires
- if f not in test.config.available_features]
+ if f not in test.config.available_features
+ or f not in test.config.target_triple]
if missing_required_features:
return (lit.Test.UNSUPPORTED,
"Test requires the following features: %s" % (
', '.join(missing_required_features),))
unsupported_features = [f for f in unsupported
- if f in test.config.available_features]
+ if f in test.config.available_features
+ or f in test.config.target_triple]
if unsupported_features:
return (lit.Test.UNSUPPORTED,
"Test is unsupported with the following features: %s" % (
', '.join(unsupported_features),))
- # Evaluate the test.
- return self._evaluate_test(test, use_verify, lit_config)
+ # Compute list of features that make a test unsupported but also check
+ # that it fails
+ missing_required_xfail_features = [f for f in requires_xfail
+ if f not in test.config.available_features
+ and f not in test.config.target_triple]
+ unsupported_xfail_features = [f for f in unsupported_xfail
+ if f in test.config.available_features
+ or f in test.config.target_triple]
- def _evaluate_test(self, test, use_verify, lit_config):
+ # Dispatch the test based on name and execute it.
name = test.path_in_suite[-1]
- source_path = test.getSourcePath()
- source_dir = os.path.dirname(source_path)
+ if name.endswith('.pass.cpp'):
+ status,report,msg = self._evaluate_pass_test(test, lit_config)
+ elif name.endswith('.fail.cpp'):
+ status,report,msg = self._evaluate_fail_test(test, lit_config, use_verify)
+ else:
+ lit_config.fatal('Unrecognized test suffix: %s' % name)
+
+ # If the test is REQUIRES-XFAIL on UNSUPPORTED-XFAIL translate the
+ # result of the test to reflect that.
+ if len(missing_required_xfail_features) != 0 or \
+ len(unsupported_xfail_features) != 0:
+ if status == lit.Test.FAIL:
+ return lit.Test.PASS,""
+ elif status == lit.Test.PASS:
+ report += "Test unexpectedly passed!"
+ return lit.Test.XPASS, report
+ else:
+ # TODO figure out what generates this
+ assert False
+ # Otherwise check for an error message and return the result
+ if msg:
+ report += msg
+ return status,report
- # Check what kind of test this is.
- assert name.endswith('.pass.cpp') or name.endswith('.fail.cpp')
- expected_compile_fail = name.endswith('.fail.cpp')
- # If this is a compile (failure) test, build it and check for failure.
- if expected_compile_fail:
- cmd = [self.cxx_under_test, '-c',
+ def _evaluate_fail_test(self, test, lit_config, use_verify):
+ source_path = test.getSourcePath()
+ cmd = [self.cxx_under_test, '-c',
'-o', '/dev/null', source_path] + self.cpp_flags
- expected_rc = 1
- if use_verify:
- cmd += ['-Xclang', '-verify']
- expected_rc = 0
- out, err, rc = self.execute_command(cmd)
- if rc == expected_rc:
- return lit.Test.PASS, ""
- else:
- report = """Command: %s\n""" % ' '.join(["'%s'" % a
- for a in cmd])
- report += """Exit Code: %d\n""" % rc
- if out:
- report += """Standard Output:\n--\n%s--""" % out
- if err:
- report += """Standard Error:\n--\n%s--""" % err
- report += "\n\nExpected compilation to fail!"
- return lit.Test.FAIL, report
+ expectedExitCode = 1
+ if use_verify:
+ cmd += ['-Xclang', '-verify']
+ expectedExitCode = 0
+ report,exitCode,_,_ = self.execute_command_and_report(cmd)
+ if exitCode == expectedExitCode:
+ return lit.Test.PASS, report, ""
else:
- exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
- exec_path = exec_file.name
- exec_file.close()
+ return lit.Test.FAIL, report, "Expected compilation to fail!"
+
+ def _evaluate_pass_test(self, test, lit_config):
+ source_path = test.getSourcePath()
+ source_dir = os.path.dirname(source_path)
+ report = ""
+
+ # If we are generating code coverage data run the compile command from
+ # the directory the .gcno and .gcda files should be put in.
+ build_cwd = None
+ if self.generate_coverage:
+ source_dir_str = str(os.path.realpath(source_dir))
+ # Subtract the test prefix "/.../libcxx/test/" from the test file directory.
+ build_cwd = source_dir_str[len(str(self.test_root)) + 1:]
+ # Set build_cwd to "/.../build-libcxx/test/coverage/test/.../path-to-test-dir"
+ build_cwd = str(os.path.join(self.coverage_root, 'test', build_cwd))
+ # Recursivly create the directory
try:
- compile_cmd = [self.cxx_under_test, '-o', exec_path,
- source_path] + self.cpp_flags + self.ld_flags
- cmd = compile_cmd
- out, err, exitCode = self.execute_command(cmd)
- if exitCode != 0:
- report = """Command: %s\n""" % ' '.join(["'%s'" % a
- for a in cmd])
- report += """Exit Code: %d\n""" % exitCode
- if out:
- report += """Standard Output:\n--\n%s--""" % out
- if err:
- report += """Standard Error:\n--\n%s--""" % err
- report += "\n\nCompilation failed unexpectedly!"
- return lit.Test.FAIL, report
+ os.makedirs(build_cwd)
+ except:
+ pass
+ exec_file = tempfile.NamedTemporaryFile(suffix="exe", delete=False)
+ exec_path = exec_file.name
+ exec_file.close()
+
+ try:
+ if self.use_clang_tidy:
+ status,report,msg = self._execute_clang_tidy_test(source_path)
+ if status != lit.Test.PASS:
+ return status,report,msg
+
+ compile_cmd = [self.cxx_under_test, '-o', exec_path,
+ source_path] + self.cpp_flags + self.ld_flags
+ if self.generate_coverage:
+ compile_cmd += self.coverage_flags
+ if self.use_scan_build:
+ cmd = [self.use_scan_build, '--status-bugs',
+ '-o', self.scan_build_output] + self.scan_build_args
+ else:
cmd = []
- if self.exec_env:
- cmd.append('env')
- cmd.extend('%s=%s' % (name, value)
- for name,value in self.exec_env.items())
- cmd.append(exec_path)
- if lit_config.useValgrind:
- cmd = lit_config.valgrindArgs + cmd
- out, err, exitCode = self.execute_command(cmd, source_dir)
- if exitCode != 0:
- report = """Compiled With: %s\n""" % \
- ' '.join(["'%s'" % a for a in compile_cmd])
- report += """Command: %s\n""" % \
- ' '.join(["'%s'" % a for a in cmd])
- report += """Exit Code: %d\n""" % exitCode
- if out:
- report += """Standard Output:\n--\n%s--""" % out
- if err:
- report += """Standard Error:\n--\n%s--""" % err
- report += "\n\nCompiled test failed unexpectedly!"
- return lit.Test.FAIL, report
- finally:
- try:
- os.remove(exec_path)
- except:
- pass
- return lit.Test.PASS, ""
+ cmd += compile_cmd
+ report,exitCode,_,_ = self.execute_command_and_report(cmd, cwd=build_cwd)
+ if exitCode != 0 or (self.use_scan_build and
+ (not os.path.exists(exec_path)
+ or os.path.getsize(exec_path) == 0)):
+ return lit.Test.FAIL, report, "Compilation failed unexpectedly!"
+
+ cmd = []
+ cmd.append(exec_path)
+ if lit_config.useValgrind:
+ cmd = lit_config.valgrindArgs + cmd
+ report,exitCode,_,_ = self.execute_command_and_report(cmd,
+ cwd=source_dir,
+ env=self.exec_env,
+ compile_cmd=compile_cmd)
+ if exitCode != 0:
+ return lit.Test.FAIL, report, "Compiled test failed unexpectedly!"
+
+ finally:
+ try:
+ os.remove(exec_path)
+ except:
+ pass
+ return lit.Test.PASS, report, ""
+
+
+ def _execute_clang_tidy_test(self, source_file):
+ cmd = [self.use_clang_tidy] + self.clang_tidy_args + [source_file]
+ cmd += ['--'] + self.cpp_flags
+ report,exitCode,out,_ = self.execute_command_and_report(cmd)
+ if exitCode != 0 or len(out) != 0:
+ return lit.Test.FAIL, report, "Compilation failed unexpectedly!"
+ return lit.Test.PASS, report, ""
class Configuration(object):
@@ -195,12 +272,21 @@
self.cxx = None
self.src_root = None
self.obj_root = None
+ self.test_root = None
self.env = {}
self.compile_flags = []
self.library_paths = []
self.link_flags = []
+ self.generate_coverage = False
+ self.coverage_flags = []
+ self.coverage_root = None
self.use_system_lib = False
self.use_clang_verify = False
+ self.use_scan_build = None
+ self.scan_build_output = None
+ self.scan_build_args = []
+ self.use_clang_tidy = None
+ self.clang_tidy_args = []
if platform.system() not in ('Darwin', 'FreeBSD', 'Linux'):
self.lit_config.fatal("unrecognized system")
@@ -217,9 +303,20 @@
conf = self.get_lit_conf(name)
if conf is None:
return None
- if conf.lower() in ('1', 'true'):
+ if conf.lower() in ('1', 'true', 'on'):
return True
- if conf.lower() in ('', '0', 'false'):
+ if conf.lower() in ('', '0', 'false', 'off'):
+ return False
+ self.lit_config.fatal(
+ "parameter '{}' should be true or false".format(name))
+
+ def get_lit_switch(self, name):
+ s = self.get_lit_conf(name)
+ if s is None:
+ return False
+ elif s.lower() in ('', '1', 'true', 'on'):
+ return True
+ elif s.lower() in ('0', 'false', 'off'):
return False
self.lit_config.fatal(
"parameter '{}' should be true or false".format(name))
@@ -229,22 +326,38 @@
self.configure_triple()
self.configure_src_root()
self.configure_obj_root()
+ self.configure_test_root()
self.configure_use_system_lib()
self.configure_use_clang_verify()
- self.configure_env()
self.configure_std_flag()
self.configure_compile_flags()
self.configure_link_flags()
self.configure_sanitizer()
+ self.configure_warnings()
+ self.configure_generate_coverage()
+ self.configure_scan_build()
+ self.configure_clang_tidy()
self.configure_features()
+ self.lit_config.note('using compile_flags: %r' % self.compile_flags)
+ self.lit_config.note('using link_flags: %r' % self.link_flags)
+ self.lit_config.note('using available_features: %r' % self.config.available_features)
def get_test_format(self):
return LibcxxTestFormat(
self.cxx,
- self.use_clang_verify,
cpp_flags=['-nostdinc++'] + self.compile_flags,
ld_flags=['-nodefaultlibs'] + self.link_flags,
- exec_env=self.env)
+ exec_env=self.env,
+ test_root=self.test_root,
+ use_verify_for_fail=self.use_clang_verify,
+ generate_coverage=self.generate_coverage,
+ coverage_flags=self.coverage_flags,
+ coverage_root=self.coverage_root,
+ use_scan_build=self.use_scan_build,
+ scan_build_output=self.scan_build_output,
+ scan_build_args=self.scan_build_args,
+ use_clang_tidy=self.use_clang_tidy,
+ clang_tidy_args=self.clang_tidy_args)
def configure_cxx(self):
# Gather various compiler parameters.
@@ -257,11 +370,10 @@
self.config.environment['PATH'])
if clangxx:
self.cxx = clangxx
- self.lit_config.note(
- "inferred cxx_under_test as: %r" % self.cxx)
if not self.cxx:
self.lit_config.fatal('must specify user parameter cxx_under_test '
'(e.g., --param=cxx_under_test=clang++)')
+ self.lit_config.note('using cxx_under_test as: %r' % self.cxx)
def configure_src_root(self):
self.src_root = self.get_lit_conf(
@@ -270,6 +382,9 @@
def configure_obj_root(self):
self.obj_root = self.get_lit_conf('libcxx_obj_root', self.src_root)
+ def configure_test_root(self):
+ self.test_root = str(os.path.realpath(os.path.join(self.src_root, 'test')))
+
def configure_use_system_lib(self):
# This test suite supports testing against either the system library or
# the locally built one; the former mode is useful for testing ABI
@@ -279,17 +394,13 @@
if self.use_system_lib is None:
# Default to testing against the locally built libc++ library.
self.use_system_lib = False
- self.lit_config.note(
- "inferred use_system_lib as: %r" % self.use_system_lib)
+ else:
+ self.lit_config.note('using use_system_lib as: %r' % self.use_system_lib)
def configure_use_clang_verify(self):
'''If set, run clang with -verify on failing tests.'''
- self.use_clang_verify = self.get_lit_bool('use_clang_verify')
- if self.use_clang_verify is None:
- # TODO: Default this to True when using clang.
- self.use_clang_verify = False
- self.lit_config.note(
- "inferred use_clang_verify as: %r" % self.use_clang_verify)
+ self.use_clang_verify = self.get_lit_switch('use_clang_verify')
+
def configure_features(self):
additional_features = self.get_lit_conf('additional_features')
@@ -359,54 +470,74 @@
self.compile_flags += ['-D_LIBCPP_HAS_NO_MONOTONIC_CLOCK']
def configure_compile_flags(self):
+ # Check if libcxx_headers is defined. If it is then use that as
+ # the location for the libcxx headers.
+ libcxx_headers = self.get_lit_conf('libcxx_headers',
+ self.src_root + '/include')
# Configure extra compiler flags.
- self.compile_flags += ['-I' + self.src_root + '/include',
+ self.compile_flags += ['-I' + libcxx_headers,
'-I' + self.src_root + '/test/support']
if sys.platform.startswith('linux'):
self.compile_flags += ['-D__STDC_FORMAT_MACROS',
'-D__STDC_LIMIT_MACROS',
'-D__STDC_CONSTANT_MACROS']
+ # Add extra compile flags as specified on the command line.
+ extra_flags = self.get_lit_conf('compile_flags')
+ if extra_flags:
+ self.compile_flags += shlex.split(extra_flags)
+
def configure_link_flags(self):
- # Configure library search paths
- abi_library_path = self.get_lit_conf('abi_library_path', '')
- self.link_flags += ['-L' + self.obj_root + '/lib']
- if not self.use_system_lib:
- self.link_flags += ['-Wl,-rpath', '-Wl,' + self.obj_root + '/lib']
+ # Configure libc++ library
+ libcxx_library = self.get_lit_conf('libcxx_library')
+ # Ignore and warn if use_system_lib is true and libcxx_library is specified.
+ if self.use_system_lib and libcxx_library:
+ self.lit_config.warning(
+ 'Conflicting options detected: libcxx_library and use_system_lib')
+ libcxx_library = None
+ # Link to the specific libc++ library if specified. Otherwise link to -lc++
+ # and add the build directory as a search path if use_system_lib is False.
+ if libcxx_library:
+ self.link_flags += [libcxx_library, '-Wl,-rpath',
+ '-Wl,' + os.path.dirname(libcxx_library)]
+ else:
+ self.link_flags += ['-lc++']
+ if not self.use_system_lib:
+ self.link_flags += ['-L' + self.obj_root + '/lib', '-Wl,-rpath',
+ '-Wl,' + self.obj_root + '/lib']
+
+ # Add a search path for the ABI library path if specified.
+ abi_library_path = self.get_lit_conf('abi_library_path')
if abi_library_path:
self.link_flags += ['-L' + abi_library_path,
'-Wl,-rpath', '-Wl,' + abi_library_path]
- # Configure libraries
- self.link_flags += ['-lc++']
- link_flags_str = self.get_lit_conf('link_flags')
- if link_flags_str is None:
- cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi')
- if cxx_abi == 'libstdc++':
- self.link_flags += ['-lstdc++']
- elif cxx_abi == 'libsupc++':
- self.link_flags += ['-lsupc++']
- elif cxx_abi == 'libcxxabi':
- self.link_flags += ['-lc++abi']
- elif cxx_abi == 'libcxxrt':
- self.link_flags += ['-lcxxrt']
- elif cxx_abi == 'none':
- pass
- else:
- self.lit_config.fatal(
- 'C++ ABI setting %s unsupported for tests' % cxx_abi)
-
- if sys.platform == 'darwin':
- self.link_flags += ['-lSystem']
- elif sys.platform.startswith('linux'):
- self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread',
- '-lrt', '-lgcc_s']
- elif sys.platform.startswith('freebsd'):
- self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s']
- else:
- self.lit_config.fatal("unrecognized system: %r" % sys.platform)
- self.lit_config.note(
- "inferred link_flags as: %r" % self.link_flags)
+ cxx_abi = self.get_lit_conf('cxx_abi', 'libcxxabi')
+ if cxx_abi == 'libstdc++':
+ self.link_flags += ['-lstdc++']
+ elif cxx_abi == 'libsupc++':
+ self.link_flags += ['-lsupc++']
+ elif cxx_abi == 'libcxxabi':
+ self.link_flags += ['-lc++abi']
+ elif cxx_abi == 'libcxxrt':
+ self.link_flags += ['-lcxxrt']
+ elif cxx_abi == 'none':
+ pass
+ else:
+ self.lit_config.fatal(
+ 'C++ ABI setting %s unsupported for tests' % cxx_abi)
+
+ if sys.platform == 'darwin':
+ self.link_flags += ['-lSystem']
+ elif sys.platform.startswith('linux'):
+ self.link_flags += ['-lgcc_eh', '-lc', '-lm', '-lpthread',
+ '-lrt', '-lgcc_s']
+ elif sys.platform.startswith('freebsd'):
+ self.link_flags += ['-lc', '-lm', '-pthread', '-lgcc_s']
+ else:
+ self.lit_config.fatal("unrecognized system: %r" % sys.platform)
+
+ link_flags_str = self.get_lit_conf('link_flags')
if link_flags_str:
self.link_flags += shlex.split(link_flags_str)
@@ -423,8 +554,8 @@
self.config.available_features.add(std)
def configure_sanitizer(self):
- san = self.get_lit_conf('llvm_use_sanitizer', '').strip()
- if san:
+ san = self.get_lit_conf('use_sanitizer').strip()
+ if san and san != 'None':
self.compile_flags += ['-fno-omit-frame-pointer']
if sys.platform.startswith('linux'):
self.link_flags += ['-ldl']
@@ -438,12 +569,67 @@
self.config.available_features.add('msan')
elif san == 'Undefined':
self.compile_flags += ['-fsanitize=undefined',
- '-fno-sanitize=vptr,function',
- '-fno-sanitize-recover']
+ '-fno-sanitize=vptr,function',
+ '-fno-sanitize-recover']
self.config.available_features.add('ubsan')
else:
self.lit_config.fatal('unsupported value for '
- 'libcxx_use_san: {0}'.format(san))
+ 'use_sanitizer: {0}'.format(san))
+
+ def configure_warnings(self):
+ enable_warnings = self.get_lit_bool('enable_warnings')
+ if enable_warnings:
+ self.lit_config.note('Enabling warnings')
+ self.compile_flags += ['-Wall', '-Wextra', '-pedantic',
+ '-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER']
+
+ def configure_generate_coverage(self):
+ self.generate_coverage = self.get_lit_bool('generate_coverage')
+ if self.generate_coverage:
+ self.coverage_flags = ['-g', '-O0', '--coverage', '-fprofile-arcs', '-ftest-coverage']
+ self.coverage_root = self.get_lit_conf('coverage_root', None)
+ if not self.coverage_root:
+ self.lit_config.fatal("'%s' is not a valid value for coverage_root" % self.coverage_root)
+ self.lit_config.note('outputing code coverage to: %s' % self.coverage_root)
+
+ def configure_scan_build(self):
+ self.use_scan_build = self.get_lit_conf('use_scan_build')
+ if self.use_scan_build is None:
+ return
+ if self.use_scan_build == '':
+ self.use_scan_build = 'scan-build'
+ self.lit_config.note(
+ 'Using scan-build: %s' % self.use_scan_build)
+ output = self.get_lit_conf('scan_build_output')
+ if output is None:
+ output = tempfile.mkdtemp(suffix='', prefix='libcxx-scan-build-')
+ self.lit_config.note('Writing scan-build output to: %s' % output)
+ self.scan_build_output = output
+ args = self.get_lit_conf('scan_build_args')
+ if args:
+ args = shlex.split(args)
+ else:
+ args = ['-analyze-headers']
+ self.lit_config.note('Using scan-build args: %r' % args)
+ self.scan_build_args = args
+
+
+ def configure_clang_tidy(self):
+ self.use_clang_tidy = self.get_lit_conf('use_clang_tidy')
+ if self.use_clang_tidy is None:
+ return
+ if self.use_clang_tidy == '':
+ self.use_clang_tidy = 'clang-tidy'
+ self.lit_config.note(
+ 'Using clang-tidy: %s' % self.use_clang_tidy)
+ args = self.get_lit_conf('clang_tidy_args')
+ if args is not None:
+ self.clang_tidy_args = shlex.split(args)
+ else:
+ self.clang_tidy_args = ["-checks=-*,llvm*,clang-analyzer*,-clang-analyzer-alpha*",
+ "-header-filter='.*'"]
+ self.lit_config.note('Using clang-tidy args: %r' % self.clang_tidy_args)
+
def configure_triple(self):
# Get or infer the target triple.
@@ -452,7 +638,7 @@
# under test.
if not self.config.target_triple:
target_triple = lit.util.capture(
- [self.cxx, '-dumpmachine']).strip()
+ [self.cxx, '-dumpmachine']).strip()
# Drop sub-major version components from the triple, because the
# current XFAIL handling expects exact matches for feature checks.
# Example: x86_64-apple-darwin14.0.0 -> x86_64-apple-darwin14
@@ -467,15 +653,8 @@
target_triple.endswith('suse-linux'):
target_triple += '-gnu'
self.config.target_triple = target_triple
- self.lit_config.note(
- "inferred target_triple as: %r" % self.config.target_triple)
-
- def configure_env(self):
- # Configure extra linker parameters.
- if sys.platform == 'darwin':
- if not self.use_system_lib:
- self.env['DYLD_LIBRARY_PATH'] = os.path.join(self.obj_root,
- 'lib')
+ self.lit_config.note(
+ "using target_triple: %r" % self.config.target_triple)
# name: The name of this test suite.
Index: test/lit.site.cfg.in
===================================================================
--- test/lit.site.cfg.in
+++ test/lit.site.cfg.in
@@ -6,8 +6,10 @@
config.python_executable = "@PYTHON_EXECUTABLE@"
config.enable_shared = @LIBCXX_ENABLE_SHARED@
config.cxx_abi = "@LIBCXX_CXX_ABI_LIBNAME@"
-config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
config.abi_library_path = "@LIBCXX_CXX_ABI_LIBRARY_PATH@"
+config.use_sanitizer = "@LLVM_USE_SANITIZER@"
+config.generate_coverage = "@LIBCXX_GENERATE_COVERAGE@"
+config.coverage_root = "@LIBCXX_COVERAGE_DIR@"
# Let the main config do the real work.
lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")
Index: www/index.html
===================================================================
--- www/index.html
+++ www/index.html
@@ -473,6 +473,7 @@
<li><a href="type_traits_design.html"><tt><type_traits></tt></a></li>
<li><a href="http://cplusplusmusings.wordpress.com/2012/07/05/clang-and-standard-libraries-on-mac-os-x/";>Excellent notes by Marshall Clow</a></li>
<li><a href="debug_mode.html">Status of debug mode</a></li>
+<li><a href="lit_usage.html">Using LIT on the command line</a></li>
</ul>
</div>
Index: www/lit_usage.html
===================================================================
--- /dev/null
+++ www/lit_usage.html
@@ -0,0 +1,227 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
+ "http://www.w3.org/TR/html4/strict.dtd";>
+<!-- Material used from: HTML 4.01 specs: http://www.w3.org/TR/html401/ -->
+<html>
+<head>
+ <META http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+ <title>Testing libc++ using LIT</title>
+ <link type="text/css" rel="stylesheet" href="menu.css">
+ <link type="text/css" rel="stylesheet" href="content.css">
+ <style>
+ .lit-option {
+ padding-top: 0.5em;
+ margin-bottom: 0.0em;
+ font-size: medium;
+ color:#2d58b7
+ }
+ .lit-option-desc {
+ display: block;
+ margin-top: 0em;
+ margin-bottom: 0em;
+ margin-left: 20px;
+ margin-right: 20px;
+ }
+ </style>
+</head>
+
+<body>
+<div id="menu">
+ <div>
+ <a href="http://llvm.org/";>LLVM Home</a>
+ </div>
+
+ <div class="submenu">
+ <label>libc++ Info</label>
+ <a href="/index.html">About</a>
+ </div>
+
+ <div class="submenu">
+ <label>Quick Links</label>
+ <a href="http://lists.cs.uiuc.edu/mailman/listinfo/cfe-dev";>cfe-dev</a>
+ <a href="http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits";>cfe-commits</a>
+ <a href="http://llvm.org/bugs/";>Bug Reports</a>
+ <a href="http://llvm.org/svn/llvm-project/libcxx/trunk/";>Browse SVN</a>
+ <a href="http://llvm.org/viewvc/llvm-project/libcxx/trunk/";>Browse ViewVC</a>
+ </div>
+</div>
+
+<div id="content">
+<!--*********************************************************************-->
+ <h1>Testing libc++ using LIT</h1>
+<!--*********************************************************************-->
+<p>
+libc++ uses LIT to configure and run it's tests. The primary way to run the
+libc++ tests is by using <code>make check-libcxx</code>. However since libc++
+can be used in any number of possible configurations it is important to
+customize the way LIT builds and runs the tests. This guide provides
+information on how to use LIT directly to test libc++.
+</p>
+<p>
+Documentation for LIT can be found
+<a href='http://llvm.org/docs/CommandGuide/lit.html'>here</a>.
+</p>
+
+<!--*********************************************************************-->
+ <h2>Getting Started</h2>
+<!--*********************************************************************-->
+<p>
+After building libc++ use the following commands before you start using LIT to
+test.
+</p>
+<ul>
+ <li><code>alias lit='python path/to/llvm/utils/lit/lit.py'</code></li>
+ <li><code>ln -s path/to/libcxx-build/test/lit.site.cfg path/to/libcxx/test/lit.site.cfg</code></li>
+</ul>
+<p>
+You can now run the libc++ tests by running:
+</p>
+<ul>
+ <li><code>cd path/to/libcxx</code></li>
+ <li><code>lit -sv ./test</code></li>
+</ul>
+<p>
+To only run a subsection of the tests use:
+<ul>
+ <li><code>lit -sv test/numerics # Run only the numeric tests</code></li>
+</ul>
+
+<!--*********************************************************************-->
+ <h2>Customization Options</h2>
+<!--*********************************************************************-->
+<p>
+libc++'s testsuite provides multiple options to configure the way the tests
+are build and run. To use these options you pass them on the LIT command line
+as <code>--param NAME</code> or <code>--param NAME=VALUE</code>. Some options
+have default values specified during CMake's configuration. Passing the option
+on the command line will override the default.
+</p>
+
+<p>
+<h3 class="lit-option">libcxx_headers=<path/to/headers></h3>
+<blockquote class="lit-option-desc">
+Specify the libc++ headers that are tested. By default the headers in the source
+tree are used.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">libcxx_library=<path/to/libc++.so></h3>
+<blockquote class="lit-option-desc">
+Specify the libc++ library that is tested. By default the library in the build
+directory is used. This option has no effect if <code>use_system_lib</code> is provided.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">use_system_lib=<bool></h3>
+<blockquote class="lit-option-desc">
+<b>Default: </b><code>False</code></br>
+Enable or disable testing against the installed version of libc++ library.
+Note: This does not use the installed headers.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">compile_flags="<list-of-args>"</h3>
+<blockquote class="lit-option-desc">
+Specify an additional list of compile flags to use.
+Note: This options should not be used to change the standard version used.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">link_flags="<list-of-args>"</h3>
+<blockquote class="lit-option-desc">
+Specify an additional list of linker flags to use.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">std=<standard version></h3>
+<blockquote class="lit-option-desc">
+<b>Values: </b><code>c++98, c++03, c++11, c++1z, c++14</code></br>
+Change the standard version used when building the tests.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">use_sanitizer=<sanitizer name></h3>
+<blockquote class="lit-option-desc">
+<b>Values: </b><code>Memory, MemoryWithOrigins, Address, Undefined, None</code></br>
+Run the tests using the given sanitizer. If <code>LLVM_USE_SANITIZER</code>
+was given when building libc++ then that sanitizer will be used by default.
+If <code>None</code> is given then no sanitizer will be used.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">enable_warnings[=<bool>]</h3>
+<blockquote class="lit-option-desc">
+<b>Default: </b><code>False</code><br>
+Enable <code>-Wall -Wextra -pedantic</code> as well as disabling
+the system header pragmas.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">use_clang_verify[=<bool>]</h3>
+<blockquote class="lit-option-desc">
+<b>Default: </b><code>False</code><br>
+Enable the use of clang verif on tests that expect a compilation error.
+This verifies that the test fails to compile for the right reason.
+</blockquote>
+</p>
+
+<h2>Scan Build Options</h2>
+
+<p>
+<h3 class="lit-option">use_scan_build[=<path/to/scan-build>]</h3>
+<blockquote class="lit-option-desc">
+When specified the tests are compiled through scan-build. The optional value is
+used to specify the scan-build executable to use. If scan-build finds a defect
+the test will fail.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">scan_build_output=<path/to/scan-build/output></h3>
+<blockquote class="lit-option-desc">
+Specify a folder in which to store the scan-build output. If scan-build is
+used without this option a temporary directory is created and reported at the
+start of the LIT run.
+</blockquote>
+</p>
+
+<p>
+<h3 class="lit-option">scan_build_args="<list-of-args>"</h3>
+<blockquote class="lit-option-desc">
+<b>Default: </b><code>-analyze-headers</code><br>
+Specify a list of args to be passed to <code>scan-build</code>.
+</blockquote>
+</p>
+
+<h2>Clang Tidy Options</h2>
+
+<p>
+<h3 class="lit-option">use_clang_tidy[=<path/to/clang-tidy></h3>
+<blockquote class="lit-option_desc">
+When specified the tests are run through clang-tidy as an extra step. The test
+will fail if clang-tidy produces any output. The option value is used to specify
+the clang-tidy executable to use.
+</blockquote>
+</p>
+
+
+<p>
+<h3 class="lit-option">clang_tidy_args="<list-of-args>"</h3>
+<blockquote class="lit-option_desc">
+<b>Default: </b><code>-checks=-*,llvm*,clang-analyzer*,-clang-analyzer-alpha*
+ -header-filter='.*'</code><br>
+Specify a list of args to be passed to <code>clang-tidy</code>.
+</blockquote>
+</p>
+
+
+</div>
+</body>
+</html>
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
