Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
Chris Bienemanwrites: > beanz updated this revision to Diff 43192. > beanz added a comment. > > One more dependency hookup fix, this one makes it so that stage2 > doesn't depend on the stage2-instrumented compiler-rt, and avoids > building it when you invoke the 'stage2' target. Please update the cache file names to better match the updated target naming scheme, but otherwise this LGTM. > > http://reviews.llvm.org/D15584 > > Files: > CMakeLists.txt > cmake/caches/PGO-stage2.cmake > cmake/caches/PGO-stage3.cmake > cmake/caches/PGO.cmake > > Index: cmake/caches/PGO.cmake > === > --- /dev/null > +++ cmake/caches/PGO.cmake > @@ -0,0 +1,17 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > + > +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") > +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") > +set(CLANG_BOOTSTRAP_TARGETS > + generate-profdata > + stage2 > + stage2-check-all > + stage2-check-llvm > + stage2-check-clang > + stage2-test-suite CACHE STRING "") > + > +set(CLANG_BOOTSTRAP_CMAKE_ARGS > + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake > + CACHE STRING "") > Index: cmake/caches/PGO-stage3.cmake > === > --- /dev/null > +++ cmake/caches/PGO-stage3.cmake > @@ -0,0 +1,2 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > Index: cmake/caches/PGO-stage2.cmake > === > --- /dev/null > +++ cmake/caches/PGO-stage2.cmake > @@ -0,0 +1,9 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > + > +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite > CACHE STRING "") > + > +set(CLANG_BOOTSTRAP_CMAKE_ARGS > + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake > + CACHE STRING "") > Index: CMakeLists.txt > === > --- CMakeLists.txt > +++ CMakeLists.txt > @@ -631,11 +631,19 @@ > >string(REGEX MATCH "stage([0-9]*)" MATCHED_STAGE "${CLANG_STAGE}") >if(MATCHED_STAGE) > -math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") > -set(NEXT_CLANG_STAGE stage${STAGE_NUM}) > +if(NOT LLVM_BUILD_INSTRUMENTED) > + math(EXPR STAGE_NUM "${CMAKE_MATCH_1} + 1") > + set(NEXT_CLANG_STAGE stage${STAGE_NUM}) > +else() > + set(NEXT_CLANG_STAGE stage${CMAKE_MATCH_1}) > +endif() >else() > set(NEXT_CLANG_STAGE bootstrap) >endif() > + > + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) > +set(NEXT_CLANG_STAGE ${NEXT_CLANG_STAGE}-instrumented) > + endif() >message(STATUS "Setting next clang stage to: ${NEXT_CLANG_STAGE}") > > > @@ -681,6 +689,26 @@ > set(RUNTIME_DEP compiler-rt) >endif() > > + set(COMPILER_OPTIONS > +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ > +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang > +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) > + > + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) > +set(PGO_DEP llvm-profdata) > +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) > + endif() > + > + if(LLVM_BUILD_INSTRUMENTED) > +set(PGO_DEP generate-profdata) > +set(PGO_OPT > -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) > +set(COMPILER_OPTIONS > + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} > + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} > + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) > +set(RUNTIME_DEP) # Don't set runtime dependencies > + endif() > + ># Find all variables that start with BOOTSTRAP_ and populate a variable > with ># them. >get_cmake_property(variableNames VARIABLES) > @@ -703,7 +731,7 @@ >endforeach() > >ExternalProject_Add(${NEXT_CLANG_STAGE} > -DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} > +DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} > PREFIX ${NEXT_CLANG_STAGE} > SOURCE_DIR ${CMAKE_SOURCE_DIR} > STAMP_DIR ${STAMP_DIR} > @@ -715,11 +743,9 @@ > -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} > ${CLANG_BOOTSTRAP_CMAKE_ARGS} > ${PASSTHROUGH_VARIABLES} > --DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ > --DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang > --DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang > --DCLANG_STAGE=${NEXT_CLANG_STAGE} > -${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} > + -DCLANG_STAGE=${NEXT_CLANG_STAGE} > +${COMPILER_OPTIONS} > +${LTO_LIBRARY} ${LTO_AR}
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
Uh, did you intend to commit this prior to Justin or others giving an LGTM? On Fri, Dec 18, 2015 at 4:59 PM Phabricator via cfe-commits < cfe-commits@lists.llvm.org> wrote: > This revision was automatically updated to reflect the committed changes. > Closed by commit rL256069: [CMake] Support a simple case for bootstrap > builds to generate PGO data (authored by cbieneman). > > Changed prior to commit: > http://reviews.llvm.org/D15584?vs=43192=43285#toc > > Repository: > rL LLVM > > http://reviews.llvm.org/D15584 > > Files: > cfe/trunk/CMakeLists.txt > cfe/trunk/cmake/caches/PGO-stage2.cmake > cfe/trunk/cmake/caches/PGO-stage3.cmake > cfe/trunk/cmake/caches/PGO.cmake > > ___ > cfe-commits mailing list > cfe-commits@lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits > ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
This revision was automatically updated to reflect the committed changes. Closed by commit rL256069: [CMake] Support a simple case for bootstrap builds to generate PGO data (authored by cbieneman). Changed prior to commit: http://reviews.llvm.org/D15584?vs=43192=43285#toc Repository: rL LLVM http://reviews.llvm.org/D15584 Files: cfe/trunk/CMakeLists.txt cfe/trunk/cmake/caches/PGO-stage2.cmake cfe/trunk/cmake/caches/PGO-stage3.cmake cfe/trunk/cmake/caches/PGO.cmake Index: cfe/trunk/cmake/caches/PGO-stage2.cmake === --- cfe/trunk/cmake/caches/PGO-stage2.cmake +++ cfe/trunk/cmake/caches/PGO-stage2.cmake @@ -0,0 +1,9 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake + CACHE STRING "") Index: cfe/trunk/cmake/caches/PGO.cmake === --- cfe/trunk/cmake/caches/PGO.cmake +++ cfe/trunk/cmake/caches/PGO.cmake @@ -0,0 +1,17 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") +set(CLANG_BOOTSTRAP_TARGETS + generate-profdata + stage2 + stage2-check-all + stage2-check-llvm + stage2-check-clang + stage2-test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake + CACHE STRING "") Index: cfe/trunk/cmake/caches/PGO-stage3.cmake === --- cfe/trunk/cmake/caches/PGO-stage3.cmake +++ cfe/trunk/cmake/caches/PGO-stage3.cmake @@ -0,0 +1,2 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") Index: cfe/trunk/CMakeLists.txt === --- cfe/trunk/CMakeLists.txt +++ cfe/trunk/CMakeLists.txt @@ -631,11 +631,19 @@ string(REGEX MATCH "stage([0-9]*)" MATCHED_STAGE "${CLANG_STAGE}") if(MATCHED_STAGE) -math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") -set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +if(NOT LLVM_BUILD_INSTRUMENTED) + math(EXPR STAGE_NUM "${CMAKE_MATCH_1} + 1") + set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +else() + set(NEXT_CLANG_STAGE stage${CMAKE_MATCH_1}) +endif() else() set(NEXT_CLANG_STAGE bootstrap) endif() + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(NEXT_CLANG_STAGE ${NEXT_CLANG_STAGE}-instrumented) + endif() message(STATUS "Setting next clang stage to: ${NEXT_CLANG_STAGE}") @@ -681,6 +689,26 @@ set(RUNTIME_DEP compiler-rt) endif() + set(COMPILER_OPTIONS +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP llvm-profdata) +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) + endif() + + if(LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP generate-profdata) +set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) +set(COMPILER_OPTIONS + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) +set(RUNTIME_DEP) # Don't set runtime dependencies + endif() + # Find all variables that start with BOOTSTRAP_ and populate a variable with # them. get_cmake_property(variableNames VARIABLES) @@ -703,7 +731,7 @@ endforeach() ExternalProject_Add(${NEXT_CLANG_STAGE} -DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} +DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} PREFIX ${NEXT_CLANG_STAGE} SOURCE_DIR ${CMAKE_SOURCE_DIR} STAMP_DIR ${STAMP_DIR} @@ -715,11 +743,9 @@ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} ${CLANG_BOOTSTRAP_CMAKE_ARGS} ${PASSTHROUGH_VARIABLES} --DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ --DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCLANG_STAGE=${NEXT_CLANG_STAGE} -${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} + -DCLANG_STAGE=${NEXT_CLANG_STAGE} +${COMPILER_OPTIONS} +${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} INSTALL_COMMAND "" STEP_TARGETS configure build ${cmake_3_4_USES_TERMINAL_OPTIONS}
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
Chris Bienemanwrites: > beanz created this revision. > beanz added reviewers: bogner, silvas, chandlerc. > beanz added a subscriber: cfe-commits. > > This patch adds support for the clang multi-stage bootstrapping to > support PGO profdata generation, and can build a 2 or 3 stage > compiler. > > With this patch applied you can configure your build directory with > the following invocation of CMake: > > cmake -G -C /cmake/caches/PGO-stage1.cmake > > > After configuration the following additional targets will be generated: > > stage2: > Builds a stage1 x86 compiler, runtime, and required tools > (llvm-config, llvm-profdata) then uses that compiler to build an > instrumented stage2 compiler. > > stage2-generate-profdata: > Depends on "stage2" and will use the stage2 compiler to generate > profdata based on the training files in /utils/perf-training > > stage3: > Depends on "stage2-generate-profdata" and will use the stage1 compiler > with the stage2 profdata to build a PGO-optimized compiler. Let's bikeshed a bit about terminology here. This isn't really what "stage3" means in the typical sense - that term is generally for a compiler built with a stage 2. What we're getting out of this process is a PGO optimized compiler, but the result is actually a stage 2 compiler. If we ever want to build another PGO'd compiler using this one and do a binary compare, *that* would be stage 3. So if we want to use the "stage" terminology we have a stage1, a training-stage2 and a optimized-stage2, or something like that. Those are a little long though, so maybe it would make sense to just call these stage1, training, and stage2. WDYT? On a related note, the way these cache files are happening there are sort of two kinds of them. The stage1 files are more-or-less user facing and use the later stage cache files in their builds. Maybe we should drop the stage1 from the user facing files and name them so that they imply what the total result of the build will be (ie, it's just PGO). > stage3-check-llvm: > Depends on stage3 and runs check-llvm using the stage3 compiler. > > stage3-check-clang: > Depends on stage3 and runs check-clang using the stage3 compiler. > > stage3-check-all: > Depends on stage3 and runs check-all using the stage3 compiler. > > stage3-test-suite: > Depends on stage3 and runs the test-suite using the stage3 compiler > (requires in-tree test-suite). > > http://reviews.llvm.org/D15584 > > Files: > CMakeLists.txt > cmake/caches/PGO-stage1.cmake > cmake/caches/PGO-stage2.cmake > cmake/caches/PGO-stage3.cmake > > Index: cmake/caches/PGO-stage3.cmake > === > --- /dev/null > +++ cmake/caches/PGO-stage3.cmake > @@ -0,0 +1,2 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") Does this actually use the generated profdata file? I can't find a place where it does that. > Index: cmake/caches/PGO-stage2.cmake > === > --- /dev/null > +++ cmake/caches/PGO-stage2.cmake > @@ -0,0 +1,9 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > + > +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite > CACHE STRING "") > + > +set(CLANG_BOOTSTRAP_CMAKE_ARGS > + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake > + CACHE STRING "") > Index: cmake/caches/PGO-stage1.cmake > === > --- /dev/null > +++ cmake/caches/PGO-stage1.cmake > @@ -0,0 +1,17 @@ > +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") > +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") > +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > + > +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") > +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") > +set(CLANG_BOOTSTRAP_TARGETS > + stage3 > + generate-profdata > + stage3-check-all > + stage3-check-llvm > + stage3-check-clang > + stage3-test-suite CACHE STRING "") > + > +set(CLANG_BOOTSTRAP_CMAKE_ARGS > + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake > + CACHE STRING "") > Index: CMakeLists.txt > === > --- CMakeLists.txt > +++ CMakeLists.txt > @@ -677,6 +677,25 @@ > CLANG_REPOSITORY_STRING > CMAKE_MAKE_PROGRAM) > > + set(COMPILER_OPTIONS > +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ > +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang > +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) > + > + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) > +set(PGO_DEP llvm-profdata) > +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) > + endif() > + > + if(LLVM_BUILD_INSTRUMENTED) > +set(PGO_DEP generate-profdata) > +set(PGO_OPT >
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
beanz updated this revision to Diff 43177. beanz added a comment. Updates based on bogner's feedback. - Target stages are now stage2-instrumented and stage2 instead of stage2 and stage3. - Renamed PGO-stage1.cmake to PGO.cmake http://reviews.llvm.org/D15584 Files: CMakeLists.txt cmake/caches/PGO-stage2.cmake cmake/caches/PGO-stage3.cmake cmake/caches/PGO.cmake Index: cmake/caches/PGO.cmake === --- /dev/null +++ cmake/caches/PGO.cmake @@ -0,0 +1,17 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") +set(CLANG_BOOTSTRAP_TARGETS + generate-profdata + stage2 + stage2-check-all + stage2-check-llvm + stage2-check-clang + stage2-test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake + CACHE STRING "") Index: cmake/caches/PGO-stage3.cmake === --- /dev/null +++ cmake/caches/PGO-stage3.cmake @@ -0,0 +1,2 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") Index: cmake/caches/PGO-stage2.cmake === --- /dev/null +++ cmake/caches/PGO-stage2.cmake @@ -0,0 +1,9 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake + CACHE STRING "") Index: CMakeLists.txt === --- CMakeLists.txt +++ CMakeLists.txt @@ -631,11 +631,19 @@ string(REGEX MATCH "stage([0-9]*)" MATCHED_STAGE "${CLANG_STAGE}") if(MATCHED_STAGE) -math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") -set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +if(NOT LLVM_BUILD_INSTRUMENTED) + math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") + set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +else() + set(NEXT_CLANG_STAGE stage${MATCHED_STAGE}) +endif() else() set(NEXT_CLANG_STAGE bootstrap) endif() + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(NEXT_CLANG_STAGE ${NEXT_CLANG_STAGE}-instrumented) + endif() message(STATUS "Setting next clang stage to: ${NEXT_CLANG_STAGE}") @@ -677,6 +685,25 @@ CLANG_REPOSITORY_STRING CMAKE_MAKE_PROGRAM) + set(COMPILER_OPTIONS +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP llvm-profdata) +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) + endif() + + if(LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP generate-profdata) +set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) +set(COMPILER_OPTIONS + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) + endif() + if(TARGET compiler-rt) set(RUNTIME_DEP compiler-rt) endif() @@ -703,7 +730,7 @@ endforeach() ExternalProject_Add(${NEXT_CLANG_STAGE} -DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} +DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} PREFIX ${NEXT_CLANG_STAGE} SOURCE_DIR ${CMAKE_SOURCE_DIR} STAMP_DIR ${STAMP_DIR} @@ -715,11 +742,9 @@ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} ${CLANG_BOOTSTRAP_CMAKE_ARGS} ${PASSTHROUGH_VARIABLES} --DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ --DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCLANG_STAGE=${NEXT_CLANG_STAGE} -${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} + -DCLANG_STAGE=${NEXT_CLANG_STAGE} +${COMPILER_OPTIONS} +${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} INSTALL_COMMAND "" STEP_TARGETS configure build ${cmake_3_4_USES_TERMINAL_OPTIONS} ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
beanz updated this revision to Diff 43183. beanz added a comment. Updating to fix a bug in my CMake regex handling that caused the targets to not be mapped up correctly. http://reviews.llvm.org/D15584 Files: CMakeLists.txt cmake/caches/PGO-stage2.cmake cmake/caches/PGO-stage3.cmake cmake/caches/PGO.cmake Index: cmake/caches/PGO.cmake === --- /dev/null +++ cmake/caches/PGO.cmake @@ -0,0 +1,17 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") +set(CLANG_BOOTSTRAP_TARGETS + generate-profdata + stage2 + stage2-check-all + stage2-check-llvm + stage2-check-clang + stage2-test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake + CACHE STRING "") Index: cmake/caches/PGO-stage3.cmake === --- /dev/null +++ cmake/caches/PGO-stage3.cmake @@ -0,0 +1,2 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") Index: cmake/caches/PGO-stage2.cmake === --- /dev/null +++ cmake/caches/PGO-stage2.cmake @@ -0,0 +1,9 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake + CACHE STRING "") Index: CMakeLists.txt === --- CMakeLists.txt +++ CMakeLists.txt @@ -631,11 +631,19 @@ string(REGEX MATCH "stage([0-9]*)" MATCHED_STAGE "${CLANG_STAGE}") if(MATCHED_STAGE) -math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") -set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +if(NOT LLVM_BUILD_INSTRUMENTED) + math(EXPR STAGE_NUM "${CMAKE_MATCH_1} + 1") + set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +else() + set(NEXT_CLANG_STAGE stage${CMAKE_MATCH_1}) +endif() else() set(NEXT_CLANG_STAGE bootstrap) endif() + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(NEXT_CLANG_STAGE ${NEXT_CLANG_STAGE}-instrumented) + endif() message(STATUS "Setting next clang stage to: ${NEXT_CLANG_STAGE}") @@ -677,6 +685,25 @@ CLANG_REPOSITORY_STRING CMAKE_MAKE_PROGRAM) + set(COMPILER_OPTIONS +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP llvm-profdata) +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) + endif() + + if(LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP generate-profdata) +set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) +set(COMPILER_OPTIONS + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) + endif() + if(TARGET compiler-rt) set(RUNTIME_DEP compiler-rt) endif() @@ -703,7 +730,7 @@ endforeach() ExternalProject_Add(${NEXT_CLANG_STAGE} -DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} +DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} PREFIX ${NEXT_CLANG_STAGE} SOURCE_DIR ${CMAKE_SOURCE_DIR} STAMP_DIR ${STAMP_DIR} @@ -715,11 +742,9 @@ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} ${CLANG_BOOTSTRAP_CMAKE_ARGS} ${PASSTHROUGH_VARIABLES} --DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ --DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCLANG_STAGE=${NEXT_CLANG_STAGE} -${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} + -DCLANG_STAGE=${NEXT_CLANG_STAGE} +${COMPILER_OPTIONS} +${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} INSTALL_COMMAND "" STEP_TARGETS configure build ${cmake_3_4_USES_TERMINAL_OPTIONS} ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
> On Dec 17, 2015, at 11:37 AM, Justin Bognerwrote: > > Chris Bieneman > writes: >> beanz created this revision. >> beanz added reviewers: bogner, silvas, chandlerc. >> beanz added a subscriber: cfe-commits. >> >> This patch adds support for the clang multi-stage bootstrapping to >> support PGO profdata generation, and can build a 2 or 3 stage >> compiler. >> >> With this patch applied you can configure your build directory with >> the following invocation of CMake: >> >> cmake -G -C /cmake/caches/PGO-stage1.cmake >> >> >> After configuration the following additional targets will be generated: >> >> stage2: >> Builds a stage1 x86 compiler, runtime, and required tools >> (llvm-config, llvm-profdata) then uses that compiler to build an >> instrumented stage2 compiler. >> >> stage2-generate-profdata: >> Depends on "stage2" and will use the stage2 compiler to generate >> profdata based on the training files in /utils/perf-training >> >> stage3: >> Depends on "stage2-generate-profdata" and will use the stage1 compiler >> with the stage2 profdata to build a PGO-optimized compiler. > > Let's bikeshed a bit about terminology here. > > This isn't really what "stage3" means in the typical sense - that term > is generally for a compiler built with a stage 2. What we're getting out > of this process is a PGO optimized compiler, but the result is actually > a stage 2 compiler. If we ever want to build another PGO'd compiler > using this one and do a binary compare, *that* would be stage 3. > > So if we want to use the "stage" terminology we have a stage1, a > training-stage2 and a optimized-stage2, or something like that. Those > are a little long though, so maybe it would make sense to just call > these stage1, training, and stage2. WDYT? How about stage2-instrumented and just stage2? Prefacing the instrumented with “stage2” lets us know what stage we’re on. That way it will continue to work if someone did either a stage1 or stage3 instrumented build. > > > On a related note, the way these cache files are happening there are > sort of two kinds of them. The stage1 files are more-or-less user facing > and use the later stage cache files in their builds. Maybe we should > drop the stage1 from the user facing files and name them so that they > imply what the total result of the build will be (ie, it's just PGO). I’ll make that change. I’m going to send out updated patches shortly caveat that I am sending them “untested”. The CMake configuration for stage1 works, and I expect it to fully work, but the many-stage builds take a long time. > >> stage3-check-llvm: >> Depends on stage3 and runs check-llvm using the stage3 compiler. >> >> stage3-check-clang: >> Depends on stage3 and runs check-clang using the stage3 compiler. >> >> stage3-check-all: >> Depends on stage3 and runs check-all using the stage3 compiler. >> >> stage3-test-suite: >> Depends on stage3 and runs the test-suite using the stage3 compiler >> (requires in-tree test-suite). >> >> http://reviews.llvm.org/D15584 >> >> Files: >> CMakeLists.txt >> cmake/caches/PGO-stage1.cmake >> cmake/caches/PGO-stage2.cmake >> cmake/caches/PGO-stage3.cmake >> >> Index: cmake/caches/PGO-stage3.cmake >> === >> --- /dev/null >> +++ cmake/caches/PGO-stage3.cmake >> @@ -0,0 +1,2 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > > Does this actually use the generated profdata file? I can't find a place > where it does that. This is actually setup in this diff in clang/CMakelists.txt. Look for PGO_OPT. > >> Index: cmake/caches/PGO-stage2.cmake >> === >> --- /dev/null >> +++ cmake/caches/PGO-stage2.cmake >> @@ -0,0 +1,9 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite >> CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake >> + CACHE STRING "") >> Index: cmake/caches/PGO-stage1.cmake >> === >> --- /dev/null >> +++ cmake/caches/PGO-stage1.cmake >> @@ -0,0 +1,17 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") >> +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") >> +set(CLANG_BOOTSTRAP_TARGETS >> + stage3 >> + generate-profdata >> + stage3-check-all >> + stage3-check-llvm >> + stage3-check-clang >> + stage3-test-suite CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C
Re: [PATCH] D15584: [CMake] Support a simple case for bootstrap builds to generate PGO data
beanz updated this revision to Diff 43192. beanz added a comment. One more dependency hookup fix, this one makes it so that stage2 doesn't depend on the stage2-instrumented compiler-rt, and avoids building it when you invoke the 'stage2' target. http://reviews.llvm.org/D15584 Files: CMakeLists.txt cmake/caches/PGO-stage2.cmake cmake/caches/PGO-stage3.cmake cmake/caches/PGO.cmake Index: cmake/caches/PGO.cmake === --- /dev/null +++ cmake/caches/PGO.cmake @@ -0,0 +1,17 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") +set(CLANG_BOOTSTRAP_TARGETS + generate-profdata + stage2 + stage2-check-all + stage2-check-llvm + stage2-check-clang + stage2-test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake + CACHE STRING "") Index: cmake/caches/PGO-stage3.cmake === --- /dev/null +++ cmake/caches/PGO-stage3.cmake @@ -0,0 +1,2 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") Index: cmake/caches/PGO-stage2.cmake === --- /dev/null +++ cmake/caches/PGO-stage2.cmake @@ -0,0 +1,9 @@ +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") + +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite CACHE STRING "") + +set(CLANG_BOOTSTRAP_CMAKE_ARGS + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake + CACHE STRING "") Index: CMakeLists.txt === --- CMakeLists.txt +++ CMakeLists.txt @@ -631,11 +631,19 @@ string(REGEX MATCH "stage([0-9]*)" MATCHED_STAGE "${CLANG_STAGE}") if(MATCHED_STAGE) -math(EXPR STAGE_NUM "${MATCHED_STAGE} + 1") -set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +if(NOT LLVM_BUILD_INSTRUMENTED) + math(EXPR STAGE_NUM "${CMAKE_MATCH_1} + 1") + set(NEXT_CLANG_STAGE stage${STAGE_NUM}) +else() + set(NEXT_CLANG_STAGE stage${CMAKE_MATCH_1}) +endif() else() set(NEXT_CLANG_STAGE bootstrap) endif() + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(NEXT_CLANG_STAGE ${NEXT_CLANG_STAGE}-instrumented) + endif() message(STATUS "Setting next clang stage to: ${NEXT_CLANG_STAGE}") @@ -681,6 +689,26 @@ set(RUNTIME_DEP compiler-rt) endif() + set(COMPILER_OPTIONS +-DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ +-DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang +-DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) + + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP llvm-profdata) +set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) + endif() + + if(LLVM_BUILD_INSTRUMENTED) +set(PGO_DEP generate-profdata) +set(PGO_OPT -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) +set(COMPILER_OPTIONS + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) +set(RUNTIME_DEP) # Don't set runtime dependencies + endif() + # Find all variables that start with BOOTSTRAP_ and populate a variable with # them. get_cmake_property(variableNames VARIABLES) @@ -703,7 +731,7 @@ endforeach() ExternalProject_Add(${NEXT_CLANG_STAGE} -DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} +DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} PREFIX ${NEXT_CLANG_STAGE} SOURCE_DIR ${CMAKE_SOURCE_DIR} STAMP_DIR ${STAMP_DIR} @@ -715,11 +743,9 @@ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} ${CLANG_BOOTSTRAP_CMAKE_ARGS} ${PASSTHROUGH_VARIABLES} --DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ --DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang --DCLANG_STAGE=${NEXT_CLANG_STAGE} -${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} + -DCLANG_STAGE=${NEXT_CLANG_STAGE} +${COMPILER_OPTIONS} +${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} INSTALL_COMMAND "" STEP_TARGETS configure build ${cmake_3_4_USES_TERMINAL_OPTIONS} ___ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits