from:"Giulio Eulisse via cfe\-commits"

[llvm] [clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-12-22 Thread Giulio Eulisse via cfe-commits


ktf wrote:

 I was carried away by the heavy ion run and other priorities. i can try to
have a look again after Christmas. Feel free to close if you prefer.

Ciao,
Giulio

On Fri, Dec 22 2023 at 9:32 AM, Vassil Vassilev ***@***.***>
wrote:

> @ktf  what is the fate of this PR?
>
> —
> Reply to this email directly, view it on GitHub
> ,
> or unsubscribe
> 
> .
> You are receiving this because you were mentioned.Message ID:
> ***@***.***>
>


https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-03 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/9] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-03 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/8] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-03 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/7] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

IIRC I saw a ~0.3 MB reduction in memory usage in my usual test. I do not see 
any performance change, although that is not particularly the focus of my 
investigation here, so I cannot exclude it (hopefully for the better).

https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/6] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits



@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }

ktf wrote:

No, I guess not. It's just my clang-tidy integration being overzealous, I guess.

```suggestion
  bool isLoaded() const { return Loaded; }
```

https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits



@@ -103,6 +103,14 @@ template  
class PagedVector {
   /// Return the size of the vector.
   [[nodiscard]] size_t size() const { return Size; }
 
+  /// Return true if the element at `Index` belongs to a page which was already
+  /// materialized, i.e., had at least one element accessed.
+  [[nodiscard]] bool isMaterialized(size_t Index) const {

ktf wrote:

Done

https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/5] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/4] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Recover performance loss after PagedVector introduction (PR #67972)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67972

>From 154f82bd0e35e9d8ad8f8812ba3eb1cf8d211003 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Mon, 2 Oct 2023 13:01:14 +0200
Subject: [PATCH 1/3] Hint for branch likelihood

---
 llvm/include/llvm/ADT/PagedVector.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/PagedVector.h 
b/llvm/include/llvm/ADT/PagedVector.h
index 667bece6d718385..f8d014c9c84dab6 100644
--- a/llvm/include/llvm/ADT/PagedVector.h
+++ b/llvm/include/llvm/ADT/PagedVector.h
@@ -84,7 +84,7 @@ template  
class PagedVector {
 assert(Index / PageSize < PageToDataPtrs.size());
 T * = PageToDataPtrs[Index / PageSize];
 // If the page was not yet allocated, allocate it.
-if (!PagePtr) {
+if (LLVM_UNLIKELY(!PagePtr)) {
   PagePtr = Allocator.getPointer()->template Allocate(PageSize);
   // We need to invoke the default constructor on all the elements of the
   // page.

>From f119cca040b2bebabff8db32394cc4650f5115d5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Mon, 2 Oct 2023 17:19:43 +0200
Subject: [PATCH 2/3] Change PageSize to 32 elements

Moves from 42 elements (automatically calculated) to 32. It should
both reduce memory usage (it does at least in my test) and avoid
the need for a division when indexing the elements.
---
 clang/include/clang/Basic/SourceManager.h | 2 +-
 clang/lib/Basic/SourceManager.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..ac077d1ab350874 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -700,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  llvm::PagedVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..a9c6453da94a3ad 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2102,7 +2102,7 @@ std::pair 
SourceManager::isInTheSameTranslationUnit(
 unsigned Offset;
 FileID ParentFID; // Used for breaking ties.
   };
-  llvm::SmallDenseMap LChain;
+  llvm::SmallDenseMap LChain;
 
   FileID Parent;
   do {

>From 684d85f0c889358fddd52ca2a805caaad025fe10 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Mon, 2 Oct 2023 17:36:09 +0200
Subject: [PATCH 3/3] fix

---
 clang/include/clang/Basic/SourceManager.h | 2 +-
 clang/lib/Basic/SourceManager.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index ac077d1ab350874..1999e1ff4300b4f 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -700,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  llvm::PagedVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index a9c6453da94a3ad..3066cc53dbfd878 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2102,7 +2102,7 @@ std::pair 
SourceManager::isInTheSameTranslationUnit(
 unsigned Offset;
 FileID ParentFID; // Used for breaking ties.
   };
-  llvm::SmallDenseMap LChain;
+  llvm::SmallDenseMap LChain;
 
   FileID Parent;
   do {

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Recover performance loss after PagedVector introduction (PR #67972)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@nikic I have pushed a new commit where I change the size of the page to 32 
elements from the current 42. It should at least get rid of a division.

https://github.com/llvm/llvm-project/pull/67972
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Recover performance loss after PagedVector introduction (PR #67972)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf edited https://github.com/llvm/llvm-project/pull/67972
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Hint for branch likelihood (PR #67972)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67972

>From 154f82bd0e35e9d8ad8f8812ba3eb1cf8d211003 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Mon, 2 Oct 2023 13:01:14 +0200
Subject: [PATCH 1/2] Hint for branch likelihood

---
 llvm/include/llvm/ADT/PagedVector.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/PagedVector.h 
b/llvm/include/llvm/ADT/PagedVector.h
index 667bece6d718385..f8d014c9c84dab6 100644
--- a/llvm/include/llvm/ADT/PagedVector.h
+++ b/llvm/include/llvm/ADT/PagedVector.h
@@ -84,7 +84,7 @@ template  
class PagedVector {
 assert(Index / PageSize < PageToDataPtrs.size());
 T * = PageToDataPtrs[Index / PageSize];
 // If the page was not yet allocated, allocate it.
-if (!PagePtr) {
+if (LLVM_UNLIKELY(!PagePtr)) {
   PagePtr = Allocator.getPointer()->template Allocate(PageSize);
   // We need to invoke the default constructor on all the elements of the
   // page.

>From f119cca040b2bebabff8db32394cc4650f5115d5 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Mon, 2 Oct 2023 17:19:43 +0200
Subject: [PATCH 2/2] Change PageSize to 32 elements

Moves from 42 elements (automatically calculated) to 32. It should
both reduce memory usage (it does at least in my test) and avoid
the need for a division when indexing the elements.
---
 clang/include/clang/Basic/SourceManager.h | 2 +-
 clang/lib/Basic/SourceManager.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..ac077d1ab350874 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -700,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  llvm::PagedVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..a9c6453da94a3ad 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2102,7 +2102,7 @@ std::pair 
SourceManager::isInTheSameTranslationUnit(
 unsigned Offset;
 FileID ParentFID; // Used for breaking ties.
   };
-  llvm::SmallDenseMap LChain;
+  llvm::SmallDenseMap LChain;
 
   FileID Parent;
   do {

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/3] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits



@@ -103,6 +103,12 @@ template  
class PagedVector {
   /// Return the size of the vector.
   [[nodiscard]] size_t size() const { return Size; }
 
+  [[nodiscard]] bool isMaterialized(size_t Index) const {

ktf wrote:

```suggestion
  /// @return true in case the element at index @a Index belongs to a page which
  /// was already materialised.
  [[nodiscard]] bool isMaterialized(size_t Index) const {
```

https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/67960

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH 1/2] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-  assert(!SLocEntryLoaded[Index]);
+  assert(!LoadedSLocEntryTable.isMaterialized(Index) ||
+ !LoadedSLocEntryTable[Index].isLoaded());
   if (ExternalSLocEntries->ReadSLocEntry(-(static_cast(Index) + 2))) {
 if (Invalid)
   *Invalid = true;
 // If the file of the SLocEntry changed we could still have loaded it.
-if (!SLocEntryLoaded[Index]) {
+if (!LoadedSLocEntryTable.isMaterialized(Index) ||
+!LoadedSLocEntryTable[Index].isLoaded()) {

[clang] Introduce paged vector (PR #66430)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@mikaelholmen see also https://github.com/llvm/llvm-project/pull/67958. I can't 
seem to create a version that avoids triggering the warning and is also 
compatible across both macOS and Windows. Some good idea would be appreciated, 
especially because I do not have a windows setup and godbolt does not seem to 
like LLVM as an external package.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

I also noticed that LoadedSLocEntryTable uses 42 elements tables, which 
probably introduces some extra divisions to do the indexing. I can try 
switching to 32 elements.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

> @ktf could you take a look at this?

Sure, I did https://github.com/llvm/llvm-project/pull/67972 with your 
suggestion, however I am not sure how I can run the benchmark myself.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@vgvassilev IIUC, this is what you were proposing in some comment in 
https://github.com/llvm/llvm-project/pull/66430.

https://github.com/llvm/llvm-project/pull/67960
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Avoid need for SLocEntryLoaded BitVector (PR #67960)

2023-10-02 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf created https://github.com/llvm/llvm-project/pull/67960

The BitVector is currently used to keep track of which entries of 
LoadedSLocEntryTable have been loaded. Such information can be stored in the 
SLocEntry itself. Moreover, thanks to the fact that LoadedSLocEntryTable is now 
a PagedVector, we can simply consider elements of unmaterialised pages as not 
loaded.

This trades reducing the number of OffsetBits by one for reducing memory usage 
of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.

>From 9fde224de6baa5b1fb3713d810ce835d4456b457 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Fri, 29 Sep 2023 08:37:57 +0200
Subject: [PATCH] Avoid need for SLocEntryLoaded BitVector

The BitVector is currently used to keep track of which entries of
LoadedSLocEntryTable have been loaded. Such information can be stored in
the SLocEntry itself. Moreover, thanks to the fact that
LoadedSLocEntryTable is now a PagedVector, we can simply consider
elements of unmaterialised pages as not loaded.

This trades reducing the number of OffsetBits by one for reducing memory
usage of the SourceManager by LoadedSLocEntryTable.size()/8 bytes.
---
 clang/include/clang/Basic/SourceManager.h | 16 ++
 clang/lib/Basic/SourceManager.cpp | 27 ++-
 llvm/include/llvm/ADT/PagedVector.h   |  6 +
 3 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index c1b24eec2759c71..d450280303d6785 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -474,9 +474,10 @@ static_assert(sizeof(FileInfo) <= sizeof(ExpansionInfo),
 /// SourceManager keeps an array of these objects, and they are uniquely
 /// identified by the FileID datatype.
 class SLocEntry {
-  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 1;
+  static constexpr int OffsetBits = 8 * sizeof(SourceLocation::UIntTy) - 2;
   SourceLocation::UIntTy Offset : OffsetBits;
   SourceLocation::UIntTy IsExpansion : 1;
+  SourceLocation::UIntTy Loaded : 1;
   union {
 FileInfo File;
 ExpansionInfo Expansion;
@@ -489,6 +490,8 @@ class SLocEntry {
 
   bool isExpansion() const { return IsExpansion; }
   bool isFile() const { return !isExpansion(); }
+  [[nodiscard]] bool isLoaded() const { return Loaded; }
+  void setLoaded(bool Value) { Loaded = Value; }
 
   const FileInfo () const {
 assert(isFile() && "Not a file SLocEntry!");
@@ -716,13 +719,7 @@ class SourceManager : public RefCountedBase 
{
   /// The highest possible offset is 2^31-1 (2^63-1 for 64-bit source
   /// locations), so CurrentLoadedOffset starts at 2^31 (2^63 resp.).
   static const SourceLocation::UIntTy MaxLoadedOffset =
-  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
-
-  /// A bitmap that indicates whether the entries of LoadedSLocEntryTable
-  /// have already been loaded from the external source.
-  ///
-  /// Same indexing as LoadedSLocEntryTable.
-  llvm::BitVector SLocEntryLoaded;
+  1ULL << (8 * sizeof(SourceLocation::UIntTy) - 2);
 
   /// An external source for source location entries.
   ExternalSLocEntrySource *ExternalSLocEntries = nullptr;
@@ -1710,7 +1707,8 @@ class SourceManager : public 
RefCountedBase {
   const SrcMgr::SLocEntry (unsigned Index,
   bool *Invalid = nullptr) const {
 assert(Index < LoadedSLocEntryTable.size() && "Invalid index");
-if (SLocEntryLoaded[Index])
+if (LoadedSLocEntryTable.isMaterialized(Index) &&
+LoadedSLocEntryTable[Index].isLoaded())
   return LoadedSLocEntryTable[Index];
 return loadSLocEntry(Index, Invalid);
   }
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 3066cc53dbfd878..20e3f1252ddf077 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -336,7 +336,6 @@ void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
   LoadedSLocEntryTable.clear();
-  SLocEntryLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
@@ -373,7 +372,8 @@ void SourceManager::initializeForReplay(const SourceManager 
) {
 
   // Ensure all SLocEntries are loaded from the external source.
   for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
-if (!Old.SLocEntryLoaded[I])
+if (!Old.LoadedSLocEntryTable.isMaterialized(I) ||
+!Old.LoadedSLocEntryTable[I].isLoaded())
   Old.loadSLocEntry(I, nullptr);
 
   // Inherit any content cache data from the old source manager.
@@ -430,12 +430,14 @@ ContentCache ::createMemBufferContentCache(
 
 const SrcMgr::SLocEntry ::loadSLocEntry(unsigned Index,
   bool *Invalid) const {
-

[clang] Introduce paged vector (PR #66430)

2023-09-30 Thread Giulio Eulisse via cfe-commits


ktf wrote:

Thanks! If I were younger I would probably know how to post a dancing banana 
emoji at this point... ;-)

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/20] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/19] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/18] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/17] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/16] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@vgvassilev Thanks, I have pushed an update with the remaining comments 
addressed. I also verified that ROOT peaks at 52 MB rather than 82.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,342 @@
+//===- llvm/unittest/ADT/PagedVectorTest.cpp 
--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// PagedVector unit tests.
+//
+//===--===//
+
+#include "llvm/ADT/PagedVector.h"
+#include "gtest/gtest.h"
+#include 
+
+namespace llvm {
+TEST(PagedVectorTest, EmptyTest) {
+  PagedVector V;
+  EXPECT_EQ(V.empty(), true);
+  EXPECT_EQ(V.size(), 0ULL);
+  EXPECT_EQ(V.capacity(), 0ULL);
+  EXPECT_EQ(V.materialized_begin().getIndex(), 0ULL);
+  EXPECT_EQ(V.materialized_end().getIndex(), 0ULL);
+  EXPECT_EQ(std::distance(V.materialized_begin(), V.materialized_end()), 0LL);
+
+  EXPECT_DEATH(V[0], "Index < Size");
+  EXPECT_DEATH(PagedVector(nullptr), "Allocator cannot be null");
+}
+
+TEST(PagedVectorTest, ExpandTest) {
+  PagedVector V;
+  V.resize(2);
+  EXPECT_EQ(V.empty(), false);
+  EXPECT_EQ(V.size(), 2ULL);
+  EXPECT_EQ(V.capacity(), 10ULL);
+  EXPECT_EQ(V.materialized_begin().getIndex(), 2ULL);
+  EXPECT_EQ(V.materialized_end().getIndex(), 2ULL);
+  EXPECT_EQ(std::distance(V.materialized_begin(), V.materialized_end()), 0LL);
+}
+
+TEST(PagedVectorTest, FullPageFillingTest) {
+  PagedVector V;
+  V.resize(10);
+  EXPECT_EQ(V.empty(), false);
+  EXPECT_EQ(V.size(), 10ULL);
+  EXPECT_EQ(V.capacity(), 10ULL);
+  for (int I = 0; I < 10; ++I) {
+V[I] = I;
+  }

ktf wrote:

Done. Including some ifs...

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,267 @@
+//===- llvm/ADT/PagedVector.h - 'Lazyly allocated' vectors *- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// This file defines the PagedVector class.
+//
+//===--===//
+#ifndef LLVM_ADT_PAGEDVECTOR_H
+#define LLVM_ADT_PAGEDVECTOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Allocator.h"
+#include 
+#include 
+
+namespace llvm {
+/// A vector that allocates memory in pages.
+///
+/// Order is kept, but memory is allocated only when one element of the page is
+/// accessed. This introduces a level of indirection, but it is useful when you
+/// have a sparsely initialised vector where the full size is allocated 
upfront.
+///
+/// As a side effect the elements are initialised later than in a normal 
vector.
+/// On the first access to one of the elements of a given page, all the 
elements
+/// of the page are initialised. This also means that the elements of the page
+/// are initialised beyond the size of the vector.
+///
+/// Similarly on destruction the elements are destroyed only when the page is
+/// not needed anymore, delaying invoking the destructor of the elements.
+///
+/// Notice that this has iterators only on materialized elements. This
+/// is deliberately done under the assumption you would dereference the 
elements
+/// while iterating, therefore materialising them and losing the gains in terms
+/// of memory usage this container provides. If you have such a use case, you
+/// probably want to use a normal std::vector or a llvm::SmallVector.
+template  class PagedVector {
+  static_assert(PageSize > 1, "PageSize must be greater than 0. Most likely "
+  "you want it to be greater than 16.");
+  /// The actual number of elements in the vector which can be accessed.
+  size_t Size = 0;
+
+  /// The position of the initial element of the page in the Data vector.
+  /// Pages are allocated contiguously in the Data vector.
+  mutable SmallVector PageToDataPtrs;
+  /// Actual page data. All the page elements are allocated on the
+  /// first access of any of the elements of the page. Elements are default
+  /// constructed and elements of the page are stored contiguously.
+  PointerIntPair Allocator;
+
+public:
+  using value_type = T;
+
+  /// Default constructor. We build our own allocator and mark it as such with
+  /// `true` in the second pair element.
+  PagedVector() : Allocator(new BumpPtrAllocator, true) {}
+  explicit PagedVector(BumpPtrAllocator *A) : Allocator(A, false) {
+assert(A != nullptr && "Allocator cannot be nullptr");

ktf wrote:

This of course should have been `assert(A`. I fixed your commit in a subsequent 
one.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/15] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/14] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/13] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/12] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/11] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


ktf wrote:

Ping @zygoloid. I have addressed the last 6 comments. I hope I did not miss any 
this time.
Ping @vgvassilev. Could you do one last pass as well and commit? I do not think 
I have commit rights. I also checked with my ROOT build and everything works as 
before and the memory gain is still there.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-29 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 01/10] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,301 @@
+//===- llvm/ADT/PagedVector.h - 'Lazyly allocated' vectors *- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// This file defines the PagedVector class.
+//
+//===--===//
+#ifndef LLVM_ADT_PAGEDVECTOR_H
+#define LLVM_ADT_PAGEDVECTOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Allocator.h"
+#include 
+#include 
+
+namespace llvm {
+/// A vector that allocates memory in pages.
+///
+/// Order is kept, but memory is allocated only when one element of the page is
+/// accessed. This introduces a level of indirection, but it is useful when you
+/// have a sparsely initialised vector where the full size is allocated 
upfront.
+///
+/// As a side effect the elements are initialised later than in a normal 
vector.
+/// On the first access to one of the elements of a given page, all the 
elements
+/// of the page are initialised. This also means that the elements of the page
+/// are initialised beyond the size of the vector.
+///
+/// Similarly on destruction the elements are destroyed only when the page is
+/// not needed anymore, delaying invoking the destructor of the elements.
+///
+/// Notice that this has iterators only on materialised elements. This
+/// is deliberately done under the assumption you would dereference the 
elements
+/// while iterating, therefore materialising them and losing the gains in terms
+/// of memory usage this container provides. If you have such a use case, you
+/// probably want to use a normal std::vector or a llvm::SmallVector.
+template  class PagedVector {
+  static_assert(PageSize > 1, "PageSize must be greater than 0. Most likely "
+  "you want it to be greater than 16.");
+  /// The actual number of elements in the vector which can be accessed.
+  size_t Size = 0;
+
+  /// The position of the initial element of the page in the Data vector.
+  /// Pages are allocated contiguously in the Data vector.
+  mutable SmallVector PageToDataPtrs;
+  /// Actual page data. All the page elements are allocated on the
+  /// first access of any of the elements of the page. Elements are default
+  /// constructed and elements of the page are stored contiguously. The order 
of
+  /// the elements however depends on the order of access of the pages.
+  PointerIntPair Allocator;
+
+  constexpr static T *InvalidPage = nullptr;
+
+public:
+  using value_type = T;
+
+  /// Default constructor. We build our own allocator and mark it as such with
+  /// `true` in the second pair element.
+  PagedVector() : Allocator(new BumpPtrAllocator, true) {}
+  PagedVector(BumpPtrAllocator *A) : Allocator(A, false) {
+assert(A != nullptr && "Allocator cannot be null");
+  }
+
+  ~PagedVector() {
+clear();
+// If we own the allocator, delete it.
+if (Allocator.getInt())
+  delete Allocator.getPointer();
+  }
+
+  // Forbid copy and move as we do not need them for the current use case.
+  PagedVector(const PagedVector &) = delete;
+  PagedVector(PagedVector &&) = delete;
+  PagedVector =(const PagedVector &) = delete;
+  PagedVector =(PagedVector &&) = delete;
+
+  /// Look up an element at position `Index`.
+  /// If the associated page is not filled, it will be filled with default
+  /// constructed elements.
+  T [](size_t Index) const {
+assert(Index < Size);
+assert(Index / PageSize < PageToDataPtrs.size());
+T * = PageToDataPtrs[Index / PageSize];
+// If the page was not yet allocated, allocate it.
+if (PagePtr == InvalidPage) {
+  T *NewPagePtr = Allocator.getPointer()->template Allocate(PageSize);
+  // We need to invoke the default constructor on all the elements of the
+  // page.
+  std::uninitialized_value_construct_n(NewPagePtr, PageSize);
+
+  PagePtr = NewPagePtr;
+}
+// Dereference the element in the page.
+return PagePtr[Index % PageSize];
+  }
+
+  /// Return the capacity of the vector. I.e. the maximum size it can be
+  /// expanded to with the resize method without allocating more pages.
+  [[nodiscard]] size_t capacity() const {
+return PageToDataPtrs.size() * PageSize;
+  }
+
+  /// Return the size of the vector. I.e. the maximum index that can be
+  /// accessed, i.e. the maximum value which was used as argument of the
+  /// resize method.
+  [[nodiscard]] size_t size() const { return Size; }
+
+  /// Resize the vector. Notice that the constructor of the elements will not
+  /// be invoked until an element of a given page is accessed, at which point
+  /// all the elements of the page will be constructed.
+  ///
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf unresolved 
https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/8] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/7] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/6] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/5] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/4] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/3] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/2] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@dwblaikie apologies. This is my first PR to LLVM and given the effects of it 
for my use case, I guess I have been a bit to enthusiastic.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From 07da9379065daab62d43ba453ba6b71f3880a089 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo *)nullptr);

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From c14aaf14cbcd292165aece38ebdae74ce92a4d32 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/5] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From c14aaf14cbcd292165aece38ebdae74ce92a4d32 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/4] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From c14aaf14cbcd292165aece38ebdae74ce92a4d32 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/3] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From c14aaf14cbcd292165aece38ebdae74ce92a4d32 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/2] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


ktf wrote:

@zygoloid Ping. I think I have addressed all the remaining comments.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From c14aaf14cbcd292165aece38ebdae74ce92a4d32 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later, as
it happens in SourceManager and ASTReader.

By doing so, sparsely accessed PagedVector can profit from reduced
memory footprint.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(), (Decl *)nullptr);
   unsigned NumIdentifiersLoaded =
   IdentifiersLoaded.size() -
   llvm::count(IdentifiersLoaded, (IdentifierInfo *)nullptr);

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ccc518093f482dae800a9c3686e37d07b8b33895 Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 282 +++
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 672 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..bb2a32ade0a0d7c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialized(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+  llvm::count(DeclsLoaded.materialized(),

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf edited https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,323 @@
+//===- llvm/ADT/PagedVector.h - 'Lazyly allocated' vectors *- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// This file defines the PagedVector class.
+//
+//===--===//
+#ifndef LLVM_ADT_PAGEDVECTOR_H
+#define LLVM_ADT_PAGEDVECTOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Allocator.h"
+#include 
+#include 
+
+namespace llvm {
+/// A vector that allocates memory in pages.
+///
+/// Order is kept, but memory is allocated only when one element of the page is
+/// accessed. This introduces a level of indirection, but it is useful when you
+/// have a sparsely initialised vector where the full size is allocated 
upfront.
+///
+/// As a side effect the elements are initialised later than in a normal 
vector.
+/// On the first access to one of the elements of a given page all, the 
elements
+/// of the page are initialised. This also means that the elements of the page
+/// are initialised beyond the size of the vector.
+///
+/// Similarly on destruction the elements are destroyed only when the page is
+/// not needed anymore, delaying invoking the destructor of the elements.
+///
+/// Notice that this does not have iterators, because if you have iterators it
+/// probably means you are going to touch all the memory in any case, so better
+/// use a std::vector in the first place.
+template  class PagedVector {
+  static_assert(PageSize > 1, "PageSize must be greater than 0. Most likely "
+  "you want it to be greater than 16.");
+  /// The actual number of element in the vector which can be accessed.
+  size_t Size = 0;
+
+  /// The position of the initial element of the page in the Data vector.
+  /// Pages are allocated contiguously in the Data vector.
+  mutable SmallVector PageToDataPtrs;

ktf wrote:

I will switch everything to nullptr and switch back.

https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,301 @@
+//===- llvm/ADT/PagedVector.h - 'Lazyly allocated' vectors *- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// This file defines the PagedVector class.
+//
+//===--===//
+#ifndef LLVM_ADT_PAGEDVECTOR_H
+#define LLVM_ADT_PAGEDVECTOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Allocator.h"
+#include 
+#include 
+
+namespace llvm {
+/// A vector that allocates memory in pages.
+///
+/// Order is kept, but memory is allocated only when one element of the page is
+/// accessed. This introduces a level of indirection, but it is useful when you
+/// have a sparsely initialised vector where the full size is allocated 
upfront.
+///
+/// As a side effect the elements are initialised later than in a normal 
vector.
+/// On the first access to one of the elements of a given page, all the 
elements
+/// of the page are initialised. This also means that the elements of the page
+/// are initialised beyond the size of the vector.
+///
+/// Similarly on destruction the elements are destroyed only when the page is
+/// not needed anymore, delaying invoking the destructor of the elements.
+///
+/// Notice that this has iterators only on materialised elements. This
+/// is deliberately done under the assumption you would dereference the 
elements
+/// while iterating, therefore materialising them and losing the gains in terms
+/// of memory usage this container provides. If you have such a use case, you
+/// probably want to use a normal std::vector or a llvm::SmallVector.
+template  class PagedVector {
+  static_assert(PageSize > 1, "PageSize must be greater than 0. Most likely "
+  "you want it to be greater than 16.");
+  /// The actual number of elements in the vector which can be accessed.
+  size_t Size = 0;
+
+  /// The position of the initial element of the page in the Data vector.
+  /// Pages are allocated contiguously in the Data vector.
+  mutable SmallVector PageToDataPtrs;
+  /// Actual page data. All the page elements are allocated on the
+  /// first access of any of the elements of the page. Elements are default
+  /// constructed and elements of the page are stored contiguously. The order 
of
+  /// the elements however depends on the order of access of the pages.
+  PointerIntPair Allocator;
+
+  constexpr static T *InvalidPage = nullptr;
+
+public:
+  using value_type = T;
+
+  /// Default constructor. We build our own allocator and mark it as such with
+  /// `true` in the second pair element.
+  PagedVector() : Allocator(new BumpPtrAllocator, true) {}
+  PagedVector(BumpPtrAllocator *A) : Allocator(A, false) {
+assert(A != nullptr && "Allocator cannot be null");
+  }
+
+  ~PagedVector() {
+clear();
+// If we own the allocator, delete it.
+if (Allocator.getInt())
+  delete Allocator.getPointer();
+  }
+
+  // Forbid copy and move as we do not need them for the current use case.
+  PagedVector(const PagedVector &) = delete;
+  PagedVector(PagedVector &&) = delete;
+  PagedVector =(const PagedVector &) = delete;
+  PagedVector =(PagedVector &&) = delete;
+
+  /// Look up an element at position `Index`.
+  /// If the associated page is not filled, it will be filled with default
+  /// constructed elements.
+  T [](size_t Index) const {
+assert(Index < Size);
+assert(Index / PageSize < PageToDataPtrs.size());
+T * = PageToDataPtrs[Index / PageSize];
+// If the page was not yet allocated, allocate it.
+if (PagePtr == InvalidPage) {
+  T *NewPagePtr = Allocator.getPointer()->template Allocate(PageSize);
+  // We need to invoke the default constructor on all the elements of the
+  // page.
+  std::uninitialized_value_construct_n(NewPagePtr, PageSize);
+
+  PagePtr = NewPagePtr;
+}
+// Dereference the element in the page.
+return PagePtr[Index % PageSize];
+  }
+
+  /// Return the capacity of the vector. I.e. the maximum size it can be
+  /// expanded to with the resize method without allocating more pages.
+  [[nodiscard]] size_t capacity() const {
+return PageToDataPtrs.size() * PageSize;
+  }
+
+  /// Return the size of the vector. I.e. the maximum index that can be
+  /// accessed, i.e. the maximum value which was used as argument of the
+  /// resize method.
+  [[nodiscard]] size_t size() const { return Size; }
+
+  /// Resize the vector. Notice that the constructor of the elements will not
+  /// be invoked until an element of a given page is accessed, at which point
+  /// all the elements of the page will be constructed.
+  ///
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits



@@ -0,0 +1,320 @@
+//===- llvm/ADT/PagedVector.h - 'Lazyly allocated' vectors *- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+//
+// This file defines the PagedVector class.
+//
+//===--===//
+#ifndef LLVM_ADT_PAGEDVECTOR_H
+#define LLVM_ADT_PAGEDVECTOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Allocator.h"
+#include 
+#include 
+
+namespace llvm {
+/// A vector that allocates memory in pages.
+///
+/// Order is kept, but memory is allocated only when one element of the page is
+/// accessed. This introduces a level of indirection, but it is useful when you
+/// have a sparsely initialised vector where the full size is allocated 
upfront.
+///
+/// As a side effect the elements are initialised later than in a normal 
vector.
+/// On the first access to one of the elements of a given page all, the 
elements
+/// of the page are initialised. This also means that the elements of the page
+/// are initialised beyond the size of the vector.
+///
+/// Similarly on destruction the elements are destroyed only when the page is
+/// not needed anymore, delaying invoking the destructor of the elements.
+///
+/// Notice that this does not have iterators, because if you have iterators it
+/// probably means you are going to touch all the memory in any case, so better
+/// use a std::vector in the first place.
+template  class PagedVector {
+  static_assert(PageSize > 1, "PageSize must be greater than 0. Most likely "
+  "you want it to be greater than 16.");
+  /// The actual number of element in the vector which can be accessed.
+  size_t Size = 0;
+
+  /// The position of the initial element of the page in the Data vector.
+  /// Pages are allocated contiguously in the Data vector.
+  mutable SmallVector PageToDataPtrs;
+  /// Actual page data. All the page elements are added to this vector on the
+  /// first access of any of the elements of the page. Elements default
+  /// constructed and elements of the page are stored contiguously. The order 
of
+  /// the elements however depends on the order of access of the pages.
+  PointerIntPair Allocator;
+
+  constexpr static T *InvalidPage = nullptr;
+
+public:
+  using value_type = T;
+
+  /// Default constructor. We build our own allocator and mark it as such with
+  /// `true` in the second pair element.
+  PagedVector() : Allocator(new BumpPtrAllocator, true) {}
+  PagedVector(BumpPtrAllocator *A) : Allocator(A, false) {
+assert(A != nullptr && "Allocator cannot be null");
+  }
+
+  ~PagedVector() {
+clear();
+// If we own the allocator, delete it.
+if (Allocator.getInt())
+  delete Allocator.getPointer();
+  }
+
+  /// Look up an element at position `Index`.
+  /// If the associated page is not filled, it will be filled with default
+  /// constructed elements. If the associated page is filled, return the
+  /// element.
+  T [](size_t Index) const {
+assert(Index < Size);
+assert(Index / PageSize < PageToDataPtrs.size());
+T * = PageToDataPtrs[Index / PageSize];
+// If the page was not yet allocated, allocate it.
+if (PagePtr == InvalidPage) {
+  T *NewPagePtr = Allocator.getPointer()->template Allocate(PageSize);
+  // We need to invoke the default constructor on all the elements of the
+  // page.
+  std::uninitialized_value_construct_n(NewPagePtr, PageSize);
+
+  PagePtr = NewPagePtr;
+}
+// Dereference the element in the page.
+return PagePtr[Index % PageSize];
+  }
+
+  /// Return the capacity of the vector. I.e. the maximum size it can be
+  /// expanded to with the resize method without allocating more pages.
+  [[nodiscard]] size_t capacity() const {
+return PageToDataPtrs.size() * PageSize;
+  }
+
+  /// Return the size of the vector. I.e. the maximum index that can be
+  /// accessed, i.e. the maximum value which was used as argument of the
+  /// resize method.
+  [[nodiscard]] size_t size() const { return Size; }
+
+  /// Resize the vector. Notice that the constructor of the elements will not
+  /// be invoked until an element of a given page is accessed, at which point
+  /// all the elements of the page will be constructed.
+  ///
+  /// If the new size is smaller than the current size, the elements of the
+  /// pages that are not needed anymore will be destroyed, however, elements of
+  /// the last page will not be destroyed.
+  ///
+  /// For these reason the usage of this vector is discouraged if you rely
+  /// on the construction / destructor of the elements to be invoked.
+  void

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/9] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/8] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/7] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/6] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/5] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/4] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/3] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf resolved https://github.com/llvm/llvm-project/pull/66430
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] Introduce paged vector (PR #66430)

2023-09-28 Thread Giulio Eulisse via cfe-commits


https://github.com/ktf updated https://github.com/llvm/llvm-project/pull/66430

>From ca0e66a95212a9a6186cd856906a47c97d67a47a Mon Sep 17 00:00:00 2001
From: Giulio Eulisse <10544+...@users.noreply.github.com>
Date: Thu, 14 Sep 2023 21:58:21 +0200
Subject: [PATCH 1/2] Introduce PagedVector class

The goal of the class is to be an (almost) drop in replacement for
SmallVector and std::vector when those are presized and filled later,
as it happens in SourceManager and ASTReader.

By splitting the actual vector in pages of the same size and allocating
the pages only when they are needed, using this containers reduces the
memory usage by a factor 4 for the cases relevant to the ALICE
experiment ROOT / cling usage.
---
 clang/include/clang/Basic/SourceManager.h |   3 +-
 clang/include/clang/Serialization/ASTReader.h |   5 +-
 clang/lib/Basic/SourceManager.cpp |  10 +-
 clang/lib/Serialization/ASTReader.cpp |   5 +-
 llvm/docs/ProgrammersManual.rst   |  34 ++
 llvm/include/llvm/ADT/PagedVector.h   | 302 
 llvm/unittests/ADT/CMakeLists.txt |   1 +
 llvm/unittests/ADT/PagedVectorTest.cpp| 342 ++
 8 files changed, 692 insertions(+), 10 deletions(-)
 create mode 100644 llvm/include/llvm/ADT/PagedVector.h
 create mode 100644 llvm/unittests/ADT/PagedVectorTest.cpp

diff --git a/clang/include/clang/Basic/SourceManager.h 
b/clang/include/clang/Basic/SourceManager.h
index 2f846502d6f3327..e37caa2252532f9 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -43,6 +43,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
@@ -699,7 +700,7 @@ class SourceManager : public RefCountedBase {
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  SmallVector LoadedSLocEntryTable;
+  llvm::PagedVector LoadedSLocEntryTable;
 
   /// The starting offset of the next local SLocEntry.
   ///
diff --git a/clang/include/clang/Serialization/ASTReader.h 
b/clang/include/clang/Serialization/ASTReader.h
index dc1eb21c27801fe..65e19c6e44cf571 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PagedVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -487,7 +488,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the type with
   /// ID = (I + 1) << FastQual::Width has already been loaded
-  std::vector TypesLoaded;
+  llvm::PagedVector TypesLoaded;
 
   using GlobalTypeMapType =
   ContinuousRangeMap;
@@ -501,7 +502,7 @@ class ASTReader
   ///
   /// When the pointer at index I is non-NULL, the declaration with ID
   /// = I + 1 has already been loaded.
-  std::vector DeclsLoaded;
+  llvm::PagedVector DeclsLoaded;
 
   using GlobalDeclMapType =
   ContinuousRangeMap;
diff --git a/clang/lib/Basic/SourceManager.cpp 
b/clang/lib/Basic/SourceManager.cpp
index 0521ac7b30339ab..7fa8b8096ac4931 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2344,11 +2344,11 @@ SourceManager::MemoryBufferSizes 
SourceManager::getMemoryBufferSizes() const {
 }
 
 size_t SourceManager::getDataStructureSizes() const {
-  size_t size = llvm::capacity_in_bytes(MemBufferInfos)
-+ llvm::capacity_in_bytes(LocalSLocEntryTable)
-+ llvm::capacity_in_bytes(LoadedSLocEntryTable)
-+ llvm::capacity_in_bytes(SLocEntryLoaded)
-+ llvm::capacity_in_bytes(FileInfos);
+  size_t size = llvm::capacity_in_bytes(MemBufferInfos) +
+llvm::capacity_in_bytes(LocalSLocEntryTable) +
+llvm::capacity_in_bytes(LoadedSLocEntryTable) +
+llvm::capacity_in_bytes(SLocEntryLoaded) +
+llvm::capacity_in_bytes(FileInfos);
 
   if (OverriddenFilesInfo)
 size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
diff --git a/clang/lib/Serialization/ASTReader.cpp 
b/clang/lib/Serialization/ASTReader.cpp
index 0952244d037a77c..badd54987af18dd 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -7944,9 +7944,10 @@ void ASTReader::PrintStats() {
   std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded =
-  TypesLoaded.size() - llvm::count(TypesLoaded, QualType());
+  TypesLoaded.size() - llvm::count(TypesLoaded.materialised(), QualType());
   unsigned NumDeclsLoaded =
-  DeclsLoaded.size() - llvm::count(DeclsLoaded, (Decl *)nullptr);
+  DeclsLoaded.size() -
+

1 2 3 4 >

1 - 100 of 303 matches

Mail list logo