--- CMakeModules/BoostTest.cmake | 204 +++++++++++++++++++++++++++ CMakeModules/BoostTestAddTests.cmake | 138 ++++++++++++++++++ qa/CMakeLists.txt | 1 + qa/common/CMakeLists.txt | 6 +- qa/eeschema/CMakeLists.txt | 2 +- qa/libs/sexpr/CMakeLists.txt | 2 +- qa/pcbnew/CMakeLists.txt | 2 +- qa/utils/kicad2step/CMakeLists.txt | 2 +- 8 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 CMakeModules/BoostTest.cmake create mode 100644 CMakeModules/BoostTestAddTests.cmake
diff --git a/CMakeModules/BoostTest.cmake b/CMakeModules/BoostTest.cmake new file mode 100644 index 000000000..ec3419694 --- /dev/null +++ b/CMakeModules/BoostTest.cmake @@ -0,0 +1,204 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +BoostTest +---------- + +This module defines functions to help use the Boost.Test infrastructure. + +The :command:`boost_test_discover_tests` discovers tests by asking the +compiled test executable to enumerate its tests. + +This commands is intended to replace use of :command:`add_test` to register +tests, and will create a separate CTest test for each Boost.Test test case. +Note that this is in some cases less efficient, as common set-up and tear-down +logic cannot be shared by multiple test cases executing in the same instance. +However, it provides more fine-grained pass/fail information to CTest, which is +usually considered as more beneficial. By default, the CTest test name is the +same as the Boost.Test name (i.e. ``suite.testcase``); see also +``TEST_PREFIX`` and ``TEST_SUFFIX``. + +.. command:: boost_test_discover_tests + + Automatically add tests with CTest by querying the compiled test executable + for available tests:: + + boost_test_discover_tests(target + [EXTRA_ARGS arg1...] + [WORKING_DIRECTORY dir] + [TEST_PREFIX prefix] + [TEST_SUFFIX suffix] + [PROPERTIES name1 value1...] + [TEST_LIST var] + [DISCOVERY_TIMEOUT seconds] + ) + + ``boost_test_discover_tests`` sets up a post-build command on the test executable + that generates the list of tests by parsing the output from running the test + with the ``--boost_test_list_tests`` argument. This ensures that the full list of + tests, including instantiations of parameterized tests, is obtained. Since + test discovery occurs at build time, it is not necessary to re-run CMake when + the list of tests changes. + However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set + in order to function in a cross-compiling environment. + + Additionally, setting properties on tests is somewhat less convenient, since + the tests are not available at CMake time. Additional test properties may be + assigned to the set of tests as a whole using the ``PROPERTIES`` option. If + more fine-grained test control is needed, custom content may be provided + through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES` + directory property. The set of discovered tests is made accessible to such a + script via the ``<target>_TESTS`` variable. + + The options are: + + ``target`` + Specifies the Boost.Test executable, which must be a known CMake + executable target. CMake will substitute the location of the built + executable when running the test. + + ``EXTRA_ARGS arg1...`` + Any extra arguments to pass on the command line to each test case. + + ``WORKING_DIRECTORY dir`` + Specifies the directory in which to run the discovered test cases. If this + option is not provided, the current binary directory is used. + + ``TEST_PREFIX prefix`` + Specifies a ``prefix`` to be prepended to the name of each discovered test + case. This can be useful when the same test executable is being used in + multiple calls to ``boost_test_discover_tests()`` but with different + ``EXTRA_ARGS``. + + ``TEST_SUFFIX suffix`` + Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of + every discovered test case. Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may + be specified. + + ``PROPERTIES name1 value1...`` + Specifies additional properties to be set on all tests discovered by this + invocation of ``boost_test_discover_tests``. + + ``TEST_LIST var`` + Make the list of tests available in the variable ``var``, rather than the + default ``<target>_TESTS``. This can be useful when the same test + executable is being used in multiple calls to ``boost_test_discover_tests()``. + Note that this variable is only available in CTest. + + ``DISCOVERY_TIMEOUT num`` + Specifies how long (in seconds) CMake will wait for the test to enumerate + available tests. If the test takes longer than this, discovery (and your + build) will fail. Most test executables will enumerate their tests very + quickly, but under some exceptional circumstances, a test may require a + longer timeout. The default is 5. See also the ``TIMEOUT`` option of + :command:`execute_process`. + + .. note:: + + In CMake versions 3.10.1 and 3.10.2, this option was called ``TIMEOUT``. + This clashed with the ``TIMEOUT`` test property, which is one of the + common properties that would be set with the ``PROPERTIES`` keyword, + usually leading to legal but unintended behavior. The keyword was + changed to ``DISCOVERY_TIMEOUT`` in CMake 3.10.3 to address this + problem. The ambiguous behavior of the ``TIMEOUT`` keyword in 3.10.1 + and 3.10.2 has not been preserved. + +#]=======================================================================] + +# Save project's policies +cmake_policy(PUSH) +cmake_policy(SET CMP0057 NEW) # if IN_LIST + +#------------------------------------------------------------------------------ +function(boost_test_discover_tests TARGET) + cmake_parse_arguments( + "" + "" + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;DISCOVERY_TIMEOUT" + "EXTRA_ARGS;PROPERTIES" + ${ARGN} + ) + + if(NOT _WORKING_DIRECTORY) + set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif() + if(NOT _TEST_LIST) + set(_TEST_LIST ${TARGET}_TESTS) + endif() + if(NOT _DISCOVERY_TIMEOUT) + set(_DISCOVERY_TIMEOUT 5) + endif() + + get_property( + has_counter + TARGET ${TARGET} + PROPERTY CTEST_DISCOVERED_TEST_COUNTER + SET + ) + if(has_counter) + get_property( + counter + TARGET ${TARGET} + PROPERTY CTEST_DISCOVERED_TEST_COUNTER + ) + math(EXPR counter "${counter} + 1") + else() + set(counter 1) + endif() + set_property( + TARGET ${TARGET} + PROPERTY CTEST_DISCOVERED_TEST_COUNTER + ${counter} + ) + + # Define rule to generate test list for aforementioned test executable + set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}[${counter}]") + set(ctest_include_file "${ctest_file_base}_include.cmake") + set(ctest_tests_file "${ctest_file_base}_tests.cmake") + get_property(crosscompiling_emulator + TARGET ${TARGET} + PROPERTY CROSSCOMPILING_EMULATOR + ) + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_EXECUTABLE=$<TARGET_FILE:${TARGET}>" + -D "TEST_EXECUTOR=${crosscompiling_emulator}" + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" + -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" + -D "TEST_PROPERTIES=${_PROPERTIES}" + -D "TEST_PREFIX=${_TEST_PREFIX}" + -D "TEST_SUFFIX=${_TEST_SUFFIX}" + -D "TEST_LIST=${_TEST_LIST}" + -D "CTEST_FILE=${ctest_tests_file}" + -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" + -P "${_BoostTest_DISCOVER_TESTS_SCRIPT}" + VERBATIM + ) + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" + "endif()\n" + ) + + # Add discovered tests to directory TEST_INCLUDE_FILES + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) + +endfunction() + +############################################################################### + +set(_BoostTest_DISCOVER_TESTS_SCRIPT + ${CMAKE_CURRENT_LIST_DIR}/BoostTestAddTests.cmake +) + +# Restore project's policies +cmake_policy(POP) diff --git a/CMakeModules/BoostTestAddTests.cmake b/CMakeModules/BoostTestAddTests.cmake new file mode 100644 index 000000000..e7fe64f53 --- /dev/null +++ b/CMakeModules/BoostTestAddTests.cmake @@ -0,0 +1,138 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +set(prefix "${TEST_PREFIX}") +set(suffix "${TEST_SUFFIX}") +set(extra_args ${TEST_EXTRA_ARGS}) +set(properties ${TEST_PROPERTIES}) +set(script) +set(suite) +set(tests) + +function(add_command NAME) + set(_args "") + foreach(_arg ${ARGN}) + if(_arg MATCHES "[^-./:a-zA-Z0-9_]") + set(_args "${_args} [==[${_arg}]==]") + else() + set(_args "${_args} ${_arg}") + endif() + endforeach() + set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) +endfunction() + +# Run test executable to get list of available tests +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable does not exist.\n" + " Path: '${TEST_EXECUTABLE}'" + ) +endif() +execute_process( + COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" --list_content + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + TIMEOUT ${TEST_DISCOVERY_TIMEOUT} + OUTPUT_VARIABLE output + ERROR_VARIABLE error + RESULT_VARIABLE result +) +if(NOT output) + # https://github.com/boostorg/test/issues/236 + set(output "${error}") + set(error "") +endif() +if(NOT ${result} EQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + string(REPLACE "\n" "\n " error "${error}") + message(FATAL_ERROR + "Error running test executable.\n" + " Path: '${TEST_EXECUTABLE}'\n" + " Result: ${result}\n" + " Output:\n" + " ${output}\n" + " Error:\n" + " ${error}\n" + ) +endif() +string(REPLACE "\n" ";" output "${output}") + +# Parse output +set(line "") +set(parent_prefix "") +set(parent_enabled TRUE) +foreach(next_line IN LISTS output) + if (NOT line MATCHES "( *)([^ *]+)(\\*)?:?(.*)") + set(line "${next_line}") + continue() + endif() + + string(LENGTH "${CMAKE_MATCH_1}" indent) + set(name "${parent_prefix}${CMAKE_MATCH_2}") + if (parent_enabled AND CMAKE_MATCH_3 STREQUAL "*") + set(enabled TRUE) + else() + set(enabled FALSE) + endif() + + string(REGEX REPLACE "^( *)?.+$" "\\1" next_indent "${next_line}") + string(LENGTH "${next_indent}" next_indent) + if(next_indent GREATER indent) + # Suite + list(INSERT scope_name 0 "${name}") + set(parent_prefix "${name}/") + + list(INSERT scope_enabled 0 ${enabled}) + set(parent_enabled ${enabled}) + else() + # Test + # add to script + add_command(add_test + "${prefix}${name}${suffix}" + ${TEST_EXECUTOR} + "${TEST_EXECUTABLE}" + "--run_test=${name}" + ${extra_args} + ) + if(NOT enabled) + add_command(set_tests_properties + "${prefix}${name}${suffix}" + PROPERTIES DISABLED TRUE + ) + endif() + add_command(set_tests_properties + "${prefix}${name}${suffix}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${properties} + ) + list(APPEND tests "${prefix}${name}${suffix}") + endif() + + while(indent GREATER next_indent) + list(REMOVE_AT scope_name 0) + if(scope_name) + list(GET scope_name 0 parent_prefix) + set(parent_prefix "${parent_prefix}/") + else() + set(parent_prefix "") + endif() + + list(REMOVE_AT scope_enabled 0) + if(scope_enabled) + list(GET scope_enabled 0 parent_enabled) + else() + set(scope_enabled TRUE) + endif() + + math(EXPR indent "${indent} - 4") + endwhile() + + set(line "${next_line}") +endforeach() + +# Create a list of all discovered tests, which users may use to e.g. set +# properties on the tests +add_command(set ${TEST_LIST} ${tests}) + +# Write CTest script +file(WRITE "${CTEST_FILE}" "${script}") diff --git a/qa/CMakeLists.txt b/qa/CMakeLists.txt index bb379e7ff..7d2beeb8e 100644 --- a/qa/CMakeLists.txt +++ b/qa/CMakeLists.txt @@ -20,6 +20,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA include(KiCadQABuildUtils) +include(BoostTest) if( KICAD_SCRIPTING_MODULES ) diff --git a/qa/common/CMakeLists.txt b/qa/common/CMakeLists.txt index 061675514..99c4e9bb7 100644 --- a/qa/common/CMakeLists.txt +++ b/qa/common/CMakeLists.txt @@ -96,6 +96,6 @@ include_directories( ${INC_AFTER} ) -kicad_add_boost_test( qa_common_eeschema common_eeschema ) -kicad_add_boost_test( qa_common_pcbnew common_pcbnew ) -kicad_add_boost_test( qa_common_gerbview qa_common_gerbview ) +boost_test_discover_tests( qa_common_eeschema ) +boost_test_discover_tests( qa_common_pcbnew ) +boost_test_discover_tests( qa_common_gerbview ) diff --git a/qa/eeschema/CMakeLists.txt b/qa/eeschema/CMakeLists.txt index 65009ca4b..09b162518 100644 --- a/qa/eeschema/CMakeLists.txt +++ b/qa/eeschema/CMakeLists.txt @@ -81,4 +81,4 @@ set_source_files_properties( eeschema_test_utils.cpp PROPERTIES COMPILE_DEFINITIONS "QA_EESCHEMA_DATA_LOCATION=(\"${CMAKE_CURRENT_SOURCE_DIR}/data\")" ) -kicad_add_boost_test( qa_eeschema eeschema ) +boost_test_discover_tests( qa_eeschema ) diff --git a/qa/libs/sexpr/CMakeLists.txt b/qa/libs/sexpr/CMakeLists.txt index bee0b8a0a..414d6c300 100644 --- a/qa/libs/sexpr/CMakeLists.txt +++ b/qa/libs/sexpr/CMakeLists.txt @@ -41,4 +41,4 @@ target_include_directories( qa_sexpr PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -kicad_add_boost_test(qa_sexpr sexpr) \ No newline at end of file +boost_test_discover_tests( qa_sexpr ) diff --git a/qa/pcbnew/CMakeLists.txt b/qa/pcbnew/CMakeLists.txt index f07ed6160..53d67c538 100644 --- a/qa/pcbnew/CMakeLists.txt +++ b/qa/pcbnew/CMakeLists.txt @@ -81,4 +81,4 @@ target_link_libraries( qa_pcbnew ${PCBNEW_EXTRA_LIBS} # -lrt must follow Boost ) -kicad_add_boost_test( qa_pcbnew pcbnew ) \ No newline at end of file +boost_test_discover_tests( qa_pcbnew ) diff --git a/qa/utils/kicad2step/CMakeLists.txt b/qa/utils/kicad2step/CMakeLists.txt index fa58e300b..f349552ae 100644 --- a/qa/utils/kicad2step/CMakeLists.txt +++ b/qa/utils/kicad2step/CMakeLists.txt @@ -44,4 +44,4 @@ target_include_directories( qa_sexpr PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) -kicad_add_boost_test( qa_kicad2step kicad2step ) \ No newline at end of file +boost_test_discover_tests( qa_kicad2step kicad2step )
_______________________________________________ Mailing list: https://launchpad.net/~kicad-developers Post to : kicad-developers@lists.launchpad.net Unsubscribe : https://launchpad.net/~kicad-developers More help : https://help.launchpad.net/ListHelp