[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Petr Hosek via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rGe678c5117710: [Support][ThinLTO] Move ThinLTO caching to 
LLVM Support library (authored by noajshu, committed by phosek).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/docs/tools/clang-formatted-files.txt
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp
  llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
  llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn

Index: llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
===
--- llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -54,6 +54,7 @@
 "COM.cpp",
 "CRC.cpp",
 "CachePruning.cpp",
+"Caching.cpp",
 "Chrono.cpp",
 "CodeGenCoverage.cpp",
 "CommandLine.cpp",
Index: llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
===
--- llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
+++ llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
@@ -22,7 +22,6 @@
 "//llvm/lib/Transforms/Utils",
   ]
   sources = [
-"Caching.cpp",
 "LTO.cpp",
 "LTOBackend.cpp",
 "LTOCodeGenerator.cpp",
Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -19,11 +19,11 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 380527.
noajshu added a comment.

Migrate references to Caching.cpp from 
llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn to [...]/Support/BUILD.gn.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/docs/tools/clang-formatted-files.txt
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp
  llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
  llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn

Index: llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
===
--- llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -54,6 +54,7 @@
 "COM.cpp",
 "CRC.cpp",
 "CachePruning.cpp",
+"Caching.cpp",
 "Chrono.cpp",
 "CodeGenCoverage.cpp",
 "CommandLine.cpp",
Index: llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
===
--- llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
+++ llvm/utils/gn/secondary/llvm/lib/LTO/BUILD.gn
@@ -22,7 +22,6 @@
 "//llvm/lib/Transforms/Utils",
   ]
   sources = [
-"Caching.cpp",
 "LTO.cpp",
 "LTOBackend.cpp",
 "LTOCodeGenerator.cpp",
Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -19,11 +19,11 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 380498.
noajshu added a comment.

Update references to LTO/Caching in Gold plugin and Clang docs; rebase against 
main.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/docs/tools/clang-formatted-files.txt
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -19,11 +19,11 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CachePruning.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Petr Hosek via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rG92b8cc52bbc8: [Support][ThinLTO] Move ThinLTO caching to 
LLVM Support library (authored by noajshu, committed by phosek).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 380039.
noajshu marked an inline comment as done.
noajshu added a comment.

rebase against main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu marked an inline comment as done.
noajshu added inline comments.



Comment at: llvm/include/llvm/Support/Caching.h:1
+//===- Caching.h - LLVM File Cache Handling Configuration 
-===//
+//

MaskRay wrote:
> There needs to be a `*- C++ -*-`.
> Can you check whether the header has 80 characters?
Thank you, nice catch @MaskRay . I just added this indicator and can confirm 
that the header has 80 characters.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 379912.
noajshu added a comment.

Add *- C++ -*- header indicator to Caching.h


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Fangrui Song via Phabricator via cfe-commits]

MaskRay accepted this revision.
MaskRay added inline comments.



Comment at: llvm/include/llvm/Support/Caching.h:1
+//===- Caching.h - LLVM File Cache Handling Configuration 
-===//
+//

There needs to be a `*- C++ -*-`.
Can you check whether the header has 80 characters?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Teresa Johnson via Phabricator via cfe-commits]

tejohnson accepted this revision.
tejohnson added a comment.
This revision is now accepted and ready to land.

lgtm


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 379787.
noajshu added a comment.

rebase against main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 379598.
noajshu added a comment.

Rebase against main.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu added a comment.

@tejohnson Thanks for your comments! From my perspective this patch is done. 
Please let me know if you have any further changes to suggest before accepting. 
Thanks!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 379104.
noajshu added a comment.

Make cache dir parameter a Twine and avoid unnecessary copy in conversion to 
SmallString.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 379095.
noajshu added a comment.

rebase against main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378787.
noajshu added a comment.

rebase against main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:36
 
+  SmallString<10> CacheName = CacheNameRef;
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {

tejohnson wrote:
> noajshu wrote:
> > noajshu wrote:
> > > tejohnson wrote:
> > > > Is this copy necessary? I believe sys::path::append takes a Twine and 
> > > > eventually makes a copy of it.
> > > Thanks, removed!
> > Thanks again for this suggestion. On second thought I am concerned about 
> > lifetime issues with this approach. The `localCache` function returns a 
> > lambda which captures the arguments by copy. The Twine and StringRef 
> > classes just point to some temporary memory which could be modified/freed 
> > before the returned lambda is invoked. Making a copy (or just running the 
> > `sys::path::append` before returning the lambda) would protect from such 
> > bugs.
> > 
> > Here is a stripped-down example of the issue I'm concerned about:
> > ```
> > #include 
> > #include 
> > #include "llvm/ADT/Twine.h"
> > 
> > using namespace llvm;
> > 
> > auto localCacheDummy(Twine FooTwine) {
> > return [=] () {
> > std::cout << FooTwine.str();
> > };
> > }
> > 
> > int main() {
> > std::string SpecialString = "hello here is a string\n";
> > auto myFunction = localCacheDummy(SpecialString);
> > myFunction();
> > SpecialString = "ok I have replaced the string contents\n";
> > myFunction();
> > return 0;
> > }
> > ```
> > This outputs:
> > ```
> > hello here is a string
> > ok I have replaced the string contents
> > ```
> > This is an issue for `StringRef` as well, so I am concerned the [[ 
> > https://github.com/llvm/llvm-project/blob/dc066888bd98c0500ca7b590317dc91ccce0fd38/llvm/lib/LTO/Caching.cpp#L31
> >  | existing caching code ]] is unsafe as well. This was probably fine when 
> > it was used solely with the usage pattern in ThinLTO. As we move it to the 
> > support library should we make it more safe? In particular, we could keep 
> > the parameters Twines but only access them in the top part of the 
> > `localCache` body outside the lambdas.
> Hmm that is a good catch. I think you are right that we should be making a 
> copy of all of these string reference parameters outside of the lambda (which 
> we then capture by copy) to address this issue.
Thanks! These explicit copies have been added at the top of localCache.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378785.
noajshu marked an inline comment as not done.
noajshu added a comment.

Make copies of string reference parameters outside the returned lambda.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Teresa Johnson via Phabricator via cfe-commits]

tejohnson added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:36
 
+  SmallString<10> CacheName = CacheNameRef;
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {

noajshu wrote:
> noajshu wrote:
> > tejohnson wrote:
> > > Is this copy necessary? I believe sys::path::append takes a Twine and 
> > > eventually makes a copy of it.
> > Thanks, removed!
> Thanks again for this suggestion. On second thought I am concerned about 
> lifetime issues with this approach. The `localCache` function returns a 
> lambda which captures the arguments by copy. The Twine and StringRef classes 
> just point to some temporary memory which could be modified/freed before the 
> returned lambda is invoked. Making a copy (or just running the 
> `sys::path::append` before returning the lambda) would protect from such bugs.
> 
> Here is a stripped-down example of the issue I'm concerned about:
> ```
> #include 
> #include 
> #include "llvm/ADT/Twine.h"
> 
> using namespace llvm;
> 
> auto localCacheDummy(Twine FooTwine) {
>   return [=] () {
>   std::cout << FooTwine.str();
>   };
> }
> 
> int main() {
>   std::string SpecialString = "hello here is a string\n";
>   auto myFunction = localCacheDummy(SpecialString);
>   myFunction();
>   SpecialString = "ok I have replaced the string contents\n";
>   myFunction();
>   return 0;
> }
> ```
> This outputs:
> ```
> hello here is a string
> ok I have replaced the string contents
> ```
> This is an issue for `StringRef` as well, so I am concerned the [[ 
> https://github.com/llvm/llvm-project/blob/dc066888bd98c0500ca7b590317dc91ccce0fd38/llvm/lib/LTO/Caching.cpp#L31
>  | existing caching code ]] is unsafe as well. This was probably fine when it 
> was used solely with the usage pattern in ThinLTO. As we move it to the 
> support library should we make it more safe? In particular, we could keep the 
> parameters Twines but only access them in the top part of the `localCache` 
> body outside the lambdas.
Hmm that is a good catch. I think you are right that we should be making a copy 
of all of these string reference parameters outside of the lambda (which we 
then capture by copy) to address this issue.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu marked 3 inline comments as not done.
noajshu added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:36
 
+  SmallString<10> CacheName = CacheNameRef;
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {

noajshu wrote:
> tejohnson wrote:
> > Is this copy necessary? I believe sys::path::append takes a Twine and 
> > eventually makes a copy of it.
> Thanks, removed!
Thanks again for this suggestion. On second thought I am concerned about 
lifetime issues with this approach. The `localCache` function returns a lambda 
which captures the arguments by copy. The Twine and StringRef classes just 
point to some temporary memory which could be modified/freed before the 
returned lambda is invoked. Making a copy (or just running the 
`sys::path::append` before returning the lambda) would protect from such bugs.

Here is a stripped-down example of the issue I'm concerned about:
```
#include 
#include 
#include "llvm/ADT/Twine.h"

using namespace llvm;

auto localCacheDummy(Twine FooTwine) {
return [=] () {
std::cout << FooTwine.str();
};
}

int main() {
std::string SpecialString = "hello here is a string\n";
auto myFunction = localCacheDummy(SpecialString);
myFunction();
SpecialString = "ok I have replaced the string contents\n";
myFunction();
return 0;
}
```
This outputs:
```
hello here is a string
ok I have replaced the string contents
```
This is an issue for `StringRef` as well, so I am concerned the [[ 
https://github.com/llvm/llvm-project/blob/dc066888bd98c0500ca7b590317dc91ccce0fd38/llvm/lib/LTO/Caching.cpp#L31
 | existing caching code ]] is unsafe as well. This was probably fine when it 
was used solely with the usage pattern in ThinLTO. As we move it to the support 
library should we make it more safe? In particular, we could keep the 
parameters Twines but only access them in the top part of the `localCache` body 
outside the lambdas.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378328.
noajshu marked 2 inline comments as done.
noajshu added a comment.

Add documentation for new parameters to localCache.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu marked 3 inline comments as done.
noajshu added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:36
 
+  SmallString<10> CacheName = CacheNameRef;
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {

tejohnson wrote:
> Is this copy necessary? I believe sys::path::append takes a Twine and 
> eventually makes a copy of it.
Thanks, removed!



Comment at: llvm/lib/Support/Caching.cpp:32
+ AddBufferFn AddBuffer,
+ StringRef CacheNameRef) {
   if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))

phosek wrote:
> Can we make this a `Twine`? I'd also make this argument the first one.
Done! Also split out the customized cache name and file prefixes, to avoid 
changing any behavior. In ThinLTO these were "ThinLTO" and "Thin" respectively.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378316.
noajshu added a comment.

Adjust cache name/prefix customization.
Move customization arguments to be first. Convert to twines, remove unnecessary 
copy. Separate customized file prefix from customized cache name (to avoid 
chaning any behavior from existing ThinLTO cache).


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
@@ -1096,7 +1095,7 @@
 
   NativeObjectCache Cache;
   if (!options::cache_dir.empty())
-Cache = check(localCache(options::cache_dir, AddBuffer));
+Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
 
   check(Lto->run(AddStream, Cache));
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Teresa Johnson via Phabricator via cfe-commits]

tejohnson added inline comments.



Comment at: llvm/include/llvm/Support/Caching.h:68
+/// already exist.
+Expected localCache(StringRef CacheDirectoryPath,
+   AddBufferFn AddBuffer,

Nit: document new parameter.



Comment at: llvm/lib/Support/Caching.cpp:36
 
+  SmallString<10> CacheName = CacheNameRef;
   return [=](unsigned Task, StringRef Key) -> AddStreamFn {

Is this copy necessary? I believe sys::path::append takes a Twine and 
eventually makes a copy of it.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Petr Hosek via Phabricator via cfe-commits]

phosek added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:32
+ AddBufferFn AddBuffer,
+ StringRef CacheNameRef) {
   if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))

Can we make this a `Twine`? I'd also make this argument the first one.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378296.
noajshu added a comment.

Rebase against main.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache(CacheDir, AddBuffer, "Thin"),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
@@ -26,13 +26,14 @@
 #endif
 
 using namespace llvm;
-using namespace llvm::lto;
 
-Expected lto::localCache(StringRef CacheDirectoryPath,
-AddBufferFn AddBuffer) {
+Expected llvm::localCache(StringRef 

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu updated this revision to Diff 378289.
noajshu added a comment.

Make file prefix and error prefix configurable.
This sets the prefix for existing use to "Thin" to avoid chaning the behavior.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
@@ -378,7 +377,8 @@
 
   NativeObjectCache Cache;
   if (!CacheDir.empty())
-Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
+Cache = check(localCache(CacheDir, AddBuffer, "Thin"),
+  "failed to create cache");
 
   check(Lto.run(AddStream, Cache), "LTO::run failed");
   return 0;
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
@@ -26,13 +26,14 @@
 #endif
 
 using namespace llvm;
-using namespace llvm::lto;
 
-Expected lto::localCache(StringRef CacheDirectoryPath,
-

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Teresa Johnson via Phabricator via cfe-commits]

tejohnson added a comment.

This lgtm but I agree with @phosek 's suggestion to configure the file prefix 
and keep it as "Thin" for ThinLTO.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Petr Hosek via Phabricator via cfe-commits]

phosek added inline comments.



Comment at: llvm/lib/Support/Caching.cpp:137
+  sys::path::append(TempFilenameModel, CacheDirectoryPath,
+"Caching-%%.tmp.o");
   Expected Temp = sys::fs::TempFile::create(

I would make this configurable via function argument and set this to `Thin` for 
the LTO uses to avoid any behavioral changes.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu added a comment.

@MaskRay Thank you for pointing this out. I just added this!
The use case is a Debuginfod client  
implementation.
The `AssetCache` in the debuginfod client revision will be replaced by the 
localCache that was implemented for ThinLTO, after this revision to move the 
caching to Support. 
ThinLTO's caching code has advantages over our own, such as the tweaks for 
Windows compatibility (example) 
.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Fangrui Song via Phabricator via cfe-commits]

MaskRay added a comment.

Can you state your intended use case (make that patch a dependent of this one 
by clicking `Edit Related Revisions` if you have such a patch) so that others 
can analyze whether code sharing is justified?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D111371/new/

https://reviews.llvm.org/D111371

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D111371: [Support][ThinLTO] Move ThinLTO caching to LLVM Support library.

[Noah Shutty via Phabricator via cfe-commits]

noajshu created this revision.
Herald added subscribers: ormris, dexonsmith, steven_wu, hiraditya, 
arichardson, inglorion, sbc100, mgorny, emaste.
Herald added a reviewer: MaskRay.
Herald added a reviewer: gkm.
Herald added a project: lld-macho.
Herald added a reviewer: lld-macho.
noajshu requested review of this revision.
Herald added subscribers: llvm-commits, cfe-commits, aheejin.
Herald added projects: clang, LLVM.

We would like to move ThinLTO’s battle-tested file caching mechanism to the 
LLVM Support library so that we can use it elsewhere in LLVM.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D111371

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  lld/COFF/LTO.cpp
  lld/ELF/LTO.cpp
  lld/MachO/LTO.cpp
  lld/wasm/LTO.cpp
  llvm/include/llvm/LTO/Caching.h
  llvm/include/llvm/LTO/LTO.h
  llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h
  llvm/include/llvm/Support/Caching.h
  llvm/lib/LTO/CMakeLists.txt
  llvm/lib/LTO/Caching.cpp
  llvm/lib/LTO/LTOCodeGenerator.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Caching.cpp
  llvm/tools/gold/gold-plugin.cpp
  llvm/tools/llvm-lto/llvm-lto.cpp
  llvm/tools/llvm-lto2/llvm-lto2.cpp

Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -19,10 +19,10 @@
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
-#include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
 #include "llvm/Remarks/HotnessThresholdParser.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -362,14 +362,13 @@
   if (HasErrors)
 return 1;
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 std::string Path = OutputFilename + "." + utostr(Task);
 
 std::error_code EC;
 auto S = std::make_unique(Path, EC, sys::fs::OF_None);
 check(EC, Path);
-return std::make_unique(std::move(S));
+return std::make_unique(std::move(S));
   };
 
   auto AddBuffer = [&](size_t Task, std::unique_ptr MB) {
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,8 +1097,7 @@
 error("writing merged module failed.");
 }
 
-auto AddStream =
-[&](size_t Task) -> std::unique_ptr {
+auto AddStream = [&](size_t Task) -> std::unique_ptr {
   std::string PartFilename = OutputFilename;
   if (Parallelism != 1)
 PartFilename += "." + utostr(Task);
@@ -1108,7 +1107,7 @@
   std::make_unique(PartFilename, EC, sys::fs::OF_None);
   if (EC)
 error("error opening the file '" + PartFilename + "': " + EC.message());
-  return std::make_unique(std::move(S));
+  return std::make_unique(std::move(S));
 };
 
 if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,12 +1081,11 @@
   size_t MaxTasks = Lto->getMaxTasks();
   std::vector, bool>> Files(MaxTasks);
 
-  auto AddStream =
-  [&](size_t Task) -> std::unique_ptr {
+  auto AddStream = [&](size_t Task) -> std::unique_ptr {
 Files[Task].second = !SaveTemps;
 int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
-return std::make_unique(
+return std::make_unique(
 std::make_unique(FD, true));
   };
 
Index: llvm/lib/Support/Caching.cpp
===
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---===//
+//===-Caching.cpp - LLVM File Cache Handling --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===--===//
 //
-// This file implements the Caching for ThinLTO.
+// This file implements the Caching used by ThinLTO.
 //
 //===--===//
 
-#include "llvm/LTO/Caching.h"
+#include "llvm/Support/Caching.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
@@ -26,10 +26,9 @@
 #endif
 
 using namespace llvm;
-using namespace llvm::lto;
 
-Expected lto::localCache(StringRef CacheDirectoryPath,
-