[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-30 Thread Amir Ayupov via cfe-commits 


@@ -11,11 +15,11 @@ if(LLVM_BUILD_INSTRUMENTED)
   add_lit_testsuite(generate-profraw "Generating clang PGO data"
 ${CMAKE_CURRENT_BINARY_DIR}/pgo-data/
 EXCLUDE_FROM_CHECK_ALL
-DEPENDS clang clear-profraw ${CLANG_PERF_TRAINING_DEPS}
+DEPENDS clang clear-profraw ${CLANG_PGO_TRAINING_DEPS}
 )
 
   add_custom_target(clear-profraw
-COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw

aaupov wrote:

Where does this new path (`${CMAKE_BINARY_DIR}/profiles/`) come from?

https://github.com/llvm/llvm-project/pull/78879
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-30 Thread Amir Ayupov via cfe-commits 


@@ -2,7 +2,7 @@ set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
 set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
 
 set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING 
"")

aaupov wrote:

Makes sense to avoid touching to prevent conflicts with 
https://github.com/llvm/llvm-project/pull/78869.

https://github.com/llvm/llvm-project/pull/78879
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-30 Thread Amir Ayupov via cfe-commits 

aaupov wrote:

I'm in favor of this change especially in context of CSSPGO profiling 
(https://github.com/llvm/llvm-project/pull/79942) and BOLT LBR profiling 
(https://github.com/llvm/llvm-project/pull/69133).

https://github.com/llvm/llvm-project/pull/78879
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-23 Thread Petr Hosek via cfe-commits 


@@ -15,7 +19,7 @@ if(LLVM_BUILD_INSTRUMENTED)
 )

petrhosek wrote:

Looks like I added that in https://reviews.llvm.org/D138974 but I'm not aware 
of it being used anywhere at the moment. I'd still prefer renaming it to 
`CLANG_PGO_TRAINING_DEPS` for consistency, especially that we're now adding it 
to cache.

https://github.com/llvm/llvm-project/pull/78879
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-23 Thread Chris B via cfe-commits 

https://github.com/llvm-beanz approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/78879
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-20 Thread via cfe-commits 

llvmbot wrote:




@llvm/pr-subscribers-clang

Author: Tom Stellard (tstellar)


Changes

 The new CLANG_PGO_TRAINING_DATA_SOURCE_DIR allows users to specify a CMake 
project to use for generating the profile data.  For example, to use the 
llvm-test-suite to generate profile data you would do:

$ cmake -G Ninja -B build -S llvm -C path to 
source/clang/cmake/caches/PGO.cmake \ 
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=path to 
llvm-test-suite \
-DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes


---
Full diff: https://github.com/llvm/llvm-project/pull/78879.diff


4 Files Affected:

- (modified) clang/cmake/caches/PGO.cmake (+1-1) 
- (modified) clang/utils/perf-training/CMakeLists.txt (+11-2) 
- (modified) clang/utils/perf-training/perf-helper.py (+9-7) 
- (modified) llvm/docs/AdvancedBuilds.rst (+23) 


``diff
diff --git a/clang/cmake/caches/PGO.cmake b/clang/cmake/caches/PGO.cmake
index e1d0585e453f825..15bc755d110d19a 100644
--- a/clang/cmake/caches/PGO.cmake
+++ b/clang/cmake/caches/PGO.cmake
@@ -2,7 +2,7 @@ set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
 set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
 
 set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING 
"")
 
 set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
 set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "")
diff --git a/clang/utils/perf-training/CMakeLists.txt 
b/clang/utils/perf-training/CMakeLists.txt
index c6d51863fb1b5c2..f9d673b2e92e775 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -1,6 +1,10 @@
+include(LLVMExternalProjectUtils)
+
 set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
   "The path to a lit testsuite containing samples for PGO and order file 
generation"
   )
+set(CLANG_PGO_TRAINING_DATA_SOURCE_DIR OFF CACHE STRING "Path to source 
directory containing cmake project with source files to use for generating pgo 
data")
+set(CLANG_PERF_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to 
build the PGO training data.")
 
 if(LLVM_BUILD_INSTRUMENTED)
   configure_lit_site_cfg(
@@ -15,7 +19,7 @@ if(LLVM_BUILD_INSTRUMENTED)
 )
 
   add_custom_target(clear-profraw
-COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw
 COMMENT "Clearing old profraw data")
 
   if(NOT LLVM_PROFDATA)
@@ -26,9 +30,14 @@ if(LLVM_BUILD_INSTRUMENTED)
 message(STATUS "To enable merging PGO data LLVM_PROFDATA has to point to 
llvm-profdata")
   else()
 add_custom_target(generate-profdata
-  COMMAND "${Python3_EXECUTABLE}" 
${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} 
${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND "${Python3_EXECUTABLE}" 
${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} 
${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR} 
${CMAKE_BINARY_DIR}/profiles/
   COMMENT "Merging profdata"
   DEPENDS generate-profraw)
+if (CLANG_PGO_TRAINING_DATA_SOURCE_DIR)
+  llvm_ExternalProject_Add(generate-profraw-external 
${CLANG_PGO_TRAINING_DATA_SOURCE_DIR}
+  USE_TOOLCHAIN EXLUDE_FROM_ALL NO_INSTALL DEPENDS 
generate-profraw)
+  add_dependencies(generate-profdata generate-profraw-external)
+endif()
   endif()
 endif()
 
diff --git a/clang/utils/perf-training/perf-helper.py 
b/clang/utils/perf-training/perf-helper.py
index 99d6ab6ef08..844aa274f049aaa 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -30,26 +30,28 @@ def findFilesWithExtension(path, extension):
 
 
 def clean(args):
-if len(args) != 2:
+if len(args) < 2:
 print(
-"Usage: %s clean  \n" % __file__
+"Usage: %s clean  \n" % __file__
 + "\tRemoves all files with extension from ."
 )
 return 1
-for filename in findFilesWithExtension(args[0], args[1]):
-os.remove(filename)
+for path in args[1:-1]:
+for filename in findFilesWithExtension(path, args[-1]):
+os.remove(filename)
 return 0
 
 
 def merge(args):
-if len(args) != 3:
+if len(args) < 3:
 print(
-"Usage: %s merge   \n" % __file__
+"Usage: %s merge   \n" % __file__
 + "\tMerges all profraw files from path into output."
 )
 return 1
 cmd = [args[0], "merge", "-o", args[1]]
-cmd.extend(findFilesWithExtension(args[2], "profraw"))
+for i in range(2, len(args)):
+cmd.extend(findFilesWithExtension(args[i], "profraw"))
 subprocess.check_call(cmd)
 return 0
 
diff

[llvm] [clang] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)

2024-01-20 Thread Tom Stellard via cfe-commits 

https://github.com/tstellar created 
https://github.com/llvm/llvm-project/pull/78879

 The new CLANG_PGO_TRAINING_DATA_SOURCE_DIR allows users to specify a CMake 
project to use for generating the profile data.  For example, to use the 
llvm-test-suite to generate profile data you would do:

$ cmake -G Ninja -B build -S llvm -C /clang/cmake/caches/PGO.cmake \ 
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR= \
-DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes


>From 0719f49ecc6dd69ae4698c3e84dbf175a1bf2ed3 Mon Sep 17 00:00:00 2001
From: Tom Stellard 
Date: Sun, 21 Jan 2024 00:25:47 +
Subject: [PATCH 1/2] [CMake][PGO] Add libunwind to list of stage1 runtimes

This fixes the build since 8f90e6937a1fac80873bb2dab5f382c82ba1ba4e
which made libcxxabi use llvm's libunwind by default.
---
 clang/cmake/caches/PGO.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/cmake/caches/PGO.cmake b/clang/cmake/caches/PGO.cmake
index e1d0585e453f825..15bc755d110d19a 100644
--- a/clang/cmake/caches/PGO.cmake
+++ b/clang/cmake/caches/PGO.cmake
@@ -2,7 +2,7 @@ set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
 set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
 
 set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING 
"")
 
 set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
 set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "")

>From 5c602233ef4e54e850f6c8a17c25968bc706b898 Mon Sep 17 00:00:00 2001
From: Tom Stellard 
Date: Sat, 6 Jan 2024 07:46:01 +
Subject: [PATCH 2/2] [CMake][PGO] Add option for using an external project to
 generate profile data.

The new CLANG_PGO_TRAINING_DATA_SOURCE_DIR allows users to specify a CMake
project to use for generating the profile data.

For example, to use the llvm-test-suite to generate profile data you
would do:

$ cmake -G Ninja -B build -S llvm -C /clang/cmake/caches/PGO.cmake \
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR= \
-DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes
---
 clang/utils/perf-training/CMakeLists.txt | 13 +++--
 clang/utils/perf-training/perf-helper.py | 16 +---
 llvm/docs/AdvancedBuilds.rst | 23 +++
 3 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/clang/utils/perf-training/CMakeLists.txt 
b/clang/utils/perf-training/CMakeLists.txt
index c6d51863fb1b5c2..f9d673b2e92e775 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -1,6 +1,10 @@
+include(LLVMExternalProjectUtils)
+
 set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
   "The path to a lit testsuite containing samples for PGO and order file 
generation"
   )
+set(CLANG_PGO_TRAINING_DATA_SOURCE_DIR OFF CACHE STRING "Path to source 
directory containing cmake project with source files to use for generating pgo 
data")
+set(CLANG_PERF_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to 
build the PGO training data.")
 
 if(LLVM_BUILD_INSTRUMENTED)
   configure_lit_site_cfg(
@@ -15,7 +19,7 @@ if(LLVM_BUILD_INSTRUMENTED)
 )
 
   add_custom_target(clear-profraw
-COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py 
clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw
 COMMENT "Clearing old profraw data")
 
   if(NOT LLVM_PROFDATA)
@@ -26,9 +30,14 @@ if(LLVM_BUILD_INSTRUMENTED)
 message(STATUS "To enable merging PGO data LLVM_PROFDATA has to point to 
llvm-profdata")
   else()
 add_custom_target(generate-profdata
-  COMMAND "${Python3_EXECUTABLE}" 
${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} 
${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR}
+  COMMAND "${Python3_EXECUTABLE}" 
${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} 
${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR} 
${CMAKE_BINARY_DIR}/profiles/
   COMMENT "Merging profdata"
   DEPENDS generate-profraw)
+if (CLANG_PGO_TRAINING_DATA_SOURCE_DIR)
+  llvm_ExternalProject_Add(generate-profraw-external 
${CLANG_PGO_TRAINING_DATA_SOURCE_DIR}
+  USE_TOOLCHAIN EXLUDE_FROM_ALL NO_INSTALL DEPENDS 
generate-profraw)
+  add_dependencies(generate-profdata generate-profraw-external)
+endif()
   endif()
 endif()
 
diff --git a/clang/utils/perf-training/perf-helper.py 
b/clang/utils/perf-training/perf-helper.py
index 99d6ab6ef08..844aa274f049aaa 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -30,26 +30,28 @@ def findFilesWithExtension(path, extension):
 
 
 def clean(args):
-if len(args) != 2:
+if len(args) < 2:
 print(
-"Usage: %s clean