[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Sam McCall via Phabricator via cfe-commits]

This revision was automatically updated to reflect the committed changes.
Closed by commit rG735ab46cb414: [clangd] Dont create as much garbage 
while building Dex index. (authored by sammccall).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D79918/new/

https://reviews.llvm.org/D79918

Files:
  clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
  clang-tools-extra/clangd/index/dex/Dex.cpp
  clang-tools-extra/clangd/index/dex/Trigram.cpp
  clang-tools-extra/clangd/index/dex/Trigram.h
  clang-tools-extra/clangd/test/Inputs/requests.json
  clang-tools-extra/clangd/unittests/DexTests.cpp

Index: clang-tools-extra/clangd/unittests/DexTests.cpp
===
--- clang-tools-extra/clangd/unittests/DexTests.cpp
+++ clang-tools-extra/clangd/unittests/DexTests.cpp
@@ -366,38 +366,46 @@
   return tokensAre(Trigrams, Token::Kind::Trigram);
 }
 
+std::vector identifierTrigramTokens(llvm::StringRef S) {
+  std::vector Trigrams;
+  generateIdentifierTrigrams(S, Trigrams);
+  std::vector Tokens;
+  for (Trigram T : Trigrams)
+Tokens.emplace_back(Token::Kind::Trigram, T.str());
+  return Tokens;
+}
+
 TEST(DexTrigrams, IdentifierTrigrams) {
-  EXPECT_THAT(generateIdentifierTrigrams("X86"),
-  trigramsAre({"x86", "x", "x8"}));
+  EXPECT_THAT(identifierTrigramTokens("X86"), trigramsAre({"x86", "x", "x8"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("nl"), trigramsAre({"nl", "n"}));
+  EXPECT_THAT(identifierTrigramTokens("nl"), trigramsAre({"nl", "n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("n"), trigramsAre({"n"}));
+  EXPECT_THAT(identifierTrigramTokens("n"), trigramsAre({"n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("clangd"),
+  EXPECT_THAT(identifierTrigramTokens("clangd"),
   trigramsAre({"c", "cl", "cla", "lan", "ang", "ngd"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("abc_def"),
+  EXPECT_THAT(identifierTrigramTokens("abc_def"),
   trigramsAre({"a", "ab", "ad", "abc", "abd", "ade", "bcd", "bde",
"cde", "def"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("a_b_c_d_e_"),
+  EXPECT_THAT(identifierTrigramTokens("a_b_c_d_e_"),
   trigramsAre({"a", "a_", "ab", "abc", "bcd", "cde"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("unique_ptr"),
+  EXPECT_THAT(identifierTrigramTokens("unique_ptr"),
   trigramsAre({"u", "un", "up", "uni", "unp", "upt", "niq", "nip",
"npt", "iqu", "iqp", "ipt", "que", "qup", "qpt",
"uep", "ept", "ptr"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("TUDecl"),
+  identifierTrigramTokens("TUDecl"),
   trigramsAre({"t", "tu", "td", "tud", "tde", "ude", "dec", "ecl"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("IsOK"),
+  EXPECT_THAT(identifierTrigramTokens("IsOK"),
   trigramsAre({"i", "is", "io", "iso", "iok", "sok"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("abc_defGhij__klm"),
+  identifierTrigramTokens("abc_defGhij__klm"),
   trigramsAre({"a",   "ab",  "ad",  "abc", "abd", "ade", "adg", "bcd",
"bde", "bdg", "cde", "cdg", "def", "deg", "dgh", "dgk",
"efg", "egh", "egk", "fgh", "fgk", "ghi", "ghk", "gkl",
Index: clang-tools-extra/clangd/test/Inputs/requests.json
===
--- clang-tools-extra/clangd/test/Inputs/requests.json
+++ clang-tools-extra/clangd/test/Inputs/requests.json
@@ -1,7 +1,7 @@
-[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Dex","RestrictForCodeCompletion":true,"Scopes":["clang::clangd::", "clang::", "clang::clangd::dex::"],"AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Variable","RestrictForCodeCompletion":true,"Scopes":[""], "AnyScope":false}]
+[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false, "PreferredTypes":[]},
+{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false, "PreferredTypes":[]},

[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Kirill Bobyrev via Phabricator via cfe-commits]

kbobyrev accepted this revision.
kbobyrev added a comment.
This revision is now accepted and ready to land.

LGTM, thank you for the patch!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D79918/new/

https://reviews.llvm.org/D79918



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Sam McCall via Phabricator via cfe-commits]

sammccall updated this revision to Diff 263967.
sammccall added a comment.

Use a faster hash function, which seems to be worth something in the 
neighbourhood of 4%.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D79918/new/

https://reviews.llvm.org/D79918

Files:
  clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
  clang-tools-extra/clangd/index/dex/Dex.cpp
  clang-tools-extra/clangd/index/dex/Trigram.cpp
  clang-tools-extra/clangd/index/dex/Trigram.h
  clang-tools-extra/clangd/test/Inputs/requests.json
  clang-tools-extra/clangd/unittests/DexTests.cpp

Index: clang-tools-extra/clangd/unittests/DexTests.cpp
===
--- clang-tools-extra/clangd/unittests/DexTests.cpp
+++ clang-tools-extra/clangd/unittests/DexTests.cpp
@@ -366,38 +366,46 @@
   return tokensAre(Trigrams, Token::Kind::Trigram);
 }
 
+std::vector identifierTrigramTokens(llvm::StringRef S) {
+  std::vector Trigrams;
+  generateIdentifierTrigrams(S, Trigrams);
+  std::vector Tokens;
+  for (Trigram T : Trigrams)
+Tokens.emplace_back(Token::Kind::Trigram, T.str());
+  return Tokens;
+}
+
 TEST(DexTrigrams, IdentifierTrigrams) {
-  EXPECT_THAT(generateIdentifierTrigrams("X86"),
-  trigramsAre({"x86", "x", "x8"}));
+  EXPECT_THAT(identifierTrigramTokens("X86"), trigramsAre({"x86", "x", "x8"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("nl"), trigramsAre({"nl", "n"}));
+  EXPECT_THAT(identifierTrigramTokens("nl"), trigramsAre({"nl", "n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("n"), trigramsAre({"n"}));
+  EXPECT_THAT(identifierTrigramTokens("n"), trigramsAre({"n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("clangd"),
+  EXPECT_THAT(identifierTrigramTokens("clangd"),
   trigramsAre({"c", "cl", "cla", "lan", "ang", "ngd"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("abc_def"),
+  EXPECT_THAT(identifierTrigramTokens("abc_def"),
   trigramsAre({"a", "ab", "ad", "abc", "abd", "ade", "bcd", "bde",
"cde", "def"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("a_b_c_d_e_"),
+  EXPECT_THAT(identifierTrigramTokens("a_b_c_d_e_"),
   trigramsAre({"a", "a_", "ab", "abc", "bcd", "cde"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("unique_ptr"),
+  EXPECT_THAT(identifierTrigramTokens("unique_ptr"),
   trigramsAre({"u", "un", "up", "uni", "unp", "upt", "niq", "nip",
"npt", "iqu", "iqp", "ipt", "que", "qup", "qpt",
"uep", "ept", "ptr"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("TUDecl"),
+  identifierTrigramTokens("TUDecl"),
   trigramsAre({"t", "tu", "td", "tud", "tde", "ude", "dec", "ecl"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("IsOK"),
+  EXPECT_THAT(identifierTrigramTokens("IsOK"),
   trigramsAre({"i", "is", "io", "iso", "iok", "sok"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("abc_defGhij__klm"),
+  identifierTrigramTokens("abc_defGhij__klm"),
   trigramsAre({"a",   "ab",  "ad",  "abc", "abd", "ade", "adg", "bcd",
"bde", "bdg", "cde", "cdg", "def", "deg", "dgh", "dgk",
"efg", "egh", "egk", "fgh", "fgk", "ghi", "ghk", "gkl",
Index: clang-tools-extra/clangd/test/Inputs/requests.json
===
--- clang-tools-extra/clangd/test/Inputs/requests.json
+++ clang-tools-extra/clangd/test/Inputs/requests.json
@@ -1,7 +1,7 @@
-[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Dex","RestrictForCodeCompletion":true,"Scopes":["clang::clangd::", "clang::", "clang::clangd::dex::"],"AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Variable","RestrictForCodeCompletion":true,"Scopes":[""], "AnyScope":false}]
+[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false, "PreferredTypes":[]},
+{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false, "PreferredTypes":[]},
+{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], 

[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Sam McCall via Phabricator via cfe-commits]

sammccall updated this revision to Diff 263961.
sammccall added a comment.

Address comments, add more docs.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D79918/new/

https://reviews.llvm.org/D79918

Files:
  clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
  clang-tools-extra/clangd/index/dex/Dex.cpp
  clang-tools-extra/clangd/index/dex/Trigram.cpp
  clang-tools-extra/clangd/index/dex/Trigram.h
  clang-tools-extra/clangd/test/Inputs/requests.json
  clang-tools-extra/clangd/unittests/DexTests.cpp

Index: clang-tools-extra/clangd/unittests/DexTests.cpp
===
--- clang-tools-extra/clangd/unittests/DexTests.cpp
+++ clang-tools-extra/clangd/unittests/DexTests.cpp
@@ -366,38 +366,46 @@
   return tokensAre(Trigrams, Token::Kind::Trigram);
 }
 
+std::vector identifierTrigramTokens(llvm::StringRef S) {
+  std::vector Trigrams;
+  generateIdentifierTrigrams(S, Trigrams);
+  std::vector Tokens;
+  for (Trigram T : Trigrams)
+Tokens.emplace_back(Token::Kind::Trigram, T.str());
+  return Tokens;
+}
+
 TEST(DexTrigrams, IdentifierTrigrams) {
-  EXPECT_THAT(generateIdentifierTrigrams("X86"),
-  trigramsAre({"x86", "x", "x8"}));
+  EXPECT_THAT(identifierTrigramTokens("X86"), trigramsAre({"x86", "x", "x8"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("nl"), trigramsAre({"nl", "n"}));
+  EXPECT_THAT(identifierTrigramTokens("nl"), trigramsAre({"nl", "n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("n"), trigramsAre({"n"}));
+  EXPECT_THAT(identifierTrigramTokens("n"), trigramsAre({"n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("clangd"),
+  EXPECT_THAT(identifierTrigramTokens("clangd"),
   trigramsAre({"c", "cl", "cla", "lan", "ang", "ngd"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("abc_def"),
+  EXPECT_THAT(identifierTrigramTokens("abc_def"),
   trigramsAre({"a", "ab", "ad", "abc", "abd", "ade", "bcd", "bde",
"cde", "def"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("a_b_c_d_e_"),
+  EXPECT_THAT(identifierTrigramTokens("a_b_c_d_e_"),
   trigramsAre({"a", "a_", "ab", "abc", "bcd", "cde"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("unique_ptr"),
+  EXPECT_THAT(identifierTrigramTokens("unique_ptr"),
   trigramsAre({"u", "un", "up", "uni", "unp", "upt", "niq", "nip",
"npt", "iqu", "iqp", "ipt", "que", "qup", "qpt",
"uep", "ept", "ptr"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("TUDecl"),
+  identifierTrigramTokens("TUDecl"),
   trigramsAre({"t", "tu", "td", "tud", "tde", "ude", "dec", "ecl"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("IsOK"),
+  EXPECT_THAT(identifierTrigramTokens("IsOK"),
   trigramsAre({"i", "is", "io", "iso", "iok", "sok"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("abc_defGhij__klm"),
+  identifierTrigramTokens("abc_defGhij__klm"),
   trigramsAre({"a",   "ab",  "ad",  "abc", "abd", "ade", "adg", "bcd",
"bde", "bdg", "cde", "cdg", "def", "deg", "dgh", "dgk",
"efg", "egh", "egk", "fgh", "fgk", "ghi", "ghk", "gkl",
Index: clang-tools-extra/clangd/test/Inputs/requests.json
===
--- clang-tools-extra/clangd/test/Inputs/requests.json
+++ clang-tools-extra/clangd/test/Inputs/requests.json
@@ -1,7 +1,7 @@
-[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Dex","RestrictForCodeCompletion":true,"Scopes":["clang::clangd::", "clang::", "clang::clangd::dex::"],"AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Variable","RestrictForCodeCompletion":true,"Scopes":[""], "AnyScope":false}]
+[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false, "PreferredTypes":[]},
+{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false, "PreferredTypes":[]},
+{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false, "PreferredTypes":[]},

[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Kirill Bobyrev via Phabricator via cfe-commits]

kbobyrev added a comment.

Wow, nice work, thank you!




Comment at: clang-tools-extra/clangd/index/dex/Dex.cpp:51
+public:
+  // Returns the tokens which are given symbol's characteristics. Currently, 
the
+  // generated tokens only contain fuzzy matching trigrams and symbol's scope,

No longer "returns": "adds/generates"?



Comment at: clang-tools-extra/clangd/index/dex/Trigram.cpp:83
+  Result.clear();
+  if (Identifier.size() < 14) {
+identifierTrigrams(Identifier, [&](Trigram T) {

It isn't immediately obvious why processing different id lengths is different. 
I'm assuming it's less expensive to find duplicates in the small vector as we 
go than to find the unique elements in the end in case of the short identifier, 
but I'd be happy to have a comment here.

Also, `14` is a magic number:

* Move it to `Trigram::SMALL_IDENTIFIER_SIZE` or some other constant?
* How did you find this number? Did you try running benchmark with different 
values and figured this one is the best for the LLVM index? If not, it would 
make sense to do that.



Comment at: clang-tools-extra/clangd/index/dex/Trigram.h:49
+  std::string str() const { return std::string(Data.data(), Data[3]); }
+  uint32_t uint() const { return llvm::bit_cast(Data); }
+  friend struct ::llvm::DenseMapInfo;

nit: maybe call it `UID` or something similar, "unsigned int" is somewhat 
surprising in this context



Comment at: clang-tools-extra/clangd/index/dex/Trigram.h:55
+
 /// Returns list of unique fuzzy-search trigrams from unqualified symbol.
 /// The trigrams give the 3-character query substrings this symbol can match.

Also doesn't "return" anymore


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D79918/new/

https://reviews.llvm.org/D79918



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D79918: [clangd] Don't create as much garbage while building Dex index.

[Sam McCall via Phabricator via cfe-commits]

sammccall created this revision.
sammccall added a reviewer: kbobyrev.
Herald added subscribers: cfe-commits, usaxena95, kadircet, arphaman, mgrang, 
jkorous, MaskRay, ilya-biryukov.
Herald added a project: clang.

The Token objects are relatively expensive and we were spending a lot of
CPU creating them for each trigram emitted. Instead, use a tiny trigram
structure until we're ready to finalize the index.

This improves the new BuildDex benchmark by 20%. This code is hot and on
the critical path in clangd: it runs after a new preamble is built.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D79918

Files:
  clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
  clang-tools-extra/clangd/index/dex/Dex.cpp
  clang-tools-extra/clangd/index/dex/Trigram.cpp
  clang-tools-extra/clangd/index/dex/Trigram.h
  clang-tools-extra/clangd/test/Inputs/requests.json
  clang-tools-extra/clangd/unittests/DexTests.cpp

Index: clang-tools-extra/clangd/unittests/DexTests.cpp
===
--- clang-tools-extra/clangd/unittests/DexTests.cpp
+++ clang-tools-extra/clangd/unittests/DexTests.cpp
@@ -366,38 +366,46 @@
   return tokensAre(Trigrams, Token::Kind::Trigram);
 }
 
+std::vector identifierTrigramTokens(llvm::StringRef S) {
+  std::vector Trigrams;
+  generateIdentifierTrigrams(S, Trigrams);
+  std::vector Tokens;
+  for (Trigram T : Trigrams)
+Tokens.emplace_back(Token::Kind::Trigram, T.str());
+  return Tokens;
+}
+
 TEST(DexTrigrams, IdentifierTrigrams) {
-  EXPECT_THAT(generateIdentifierTrigrams("X86"),
-  trigramsAre({"x86", "x", "x8"}));
+  EXPECT_THAT(identifierTrigramTokens("X86"), trigramsAre({"x86", "x", "x8"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("nl"), trigramsAre({"nl", "n"}));
+  EXPECT_THAT(identifierTrigramTokens("nl"), trigramsAre({"nl", "n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("n"), trigramsAre({"n"}));
+  EXPECT_THAT(identifierTrigramTokens("n"), trigramsAre({"n"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("clangd"),
+  EXPECT_THAT(identifierTrigramTokens("clangd"),
   trigramsAre({"c", "cl", "cla", "lan", "ang", "ngd"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("abc_def"),
+  EXPECT_THAT(identifierTrigramTokens("abc_def"),
   trigramsAre({"a", "ab", "ad", "abc", "abd", "ade", "bcd", "bde",
"cde", "def"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("a_b_c_d_e_"),
+  EXPECT_THAT(identifierTrigramTokens("a_b_c_d_e_"),
   trigramsAre({"a", "a_", "ab", "abc", "bcd", "cde"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("unique_ptr"),
+  EXPECT_THAT(identifierTrigramTokens("unique_ptr"),
   trigramsAre({"u", "un", "up", "uni", "unp", "upt", "niq", "nip",
"npt", "iqu", "iqp", "ipt", "que", "qup", "qpt",
"uep", "ept", "ptr"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("TUDecl"),
+  identifierTrigramTokens("TUDecl"),
   trigramsAre({"t", "tu", "td", "tud", "tde", "ude", "dec", "ecl"}));
 
-  EXPECT_THAT(generateIdentifierTrigrams("IsOK"),
+  EXPECT_THAT(identifierTrigramTokens("IsOK"),
   trigramsAre({"i", "is", "io", "iso", "iok", "sok"}));
 
   EXPECT_THAT(
-  generateIdentifierTrigrams("abc_defGhij__klm"),
+  identifierTrigramTokens("abc_defGhij__klm"),
   trigramsAre({"a",   "ab",  "ad",  "abc", "abd", "ade", "adg", "bcd",
"bde", "bdg", "cde", "cdg", "def", "deg", "dgh", "dgk",
"efg", "egh", "egk", "fgh", "fgk", "ghi", "ghk", "gkl",
Index: clang-tools-extra/clangd/test/Inputs/requests.json
===
--- clang-tools-extra/clangd/test/Inputs/requests.json
+++ clang-tools-extra/clangd/test/Inputs/requests.json
@@ -1,7 +1,7 @@
-[{"Limit":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Scopes":["clang::"], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"s","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sy","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"sys","RestrictForCodeCompletion":true,"Scopes":["llvm::", ""], "AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Dex","RestrictForCodeCompletion":true,"Scopes":["clang::clangd::", "clang::", "clang::clangd::dex::"],"AnyScope":false},
-{"Limit":100,"ProximityPaths":[],"Query":"Variable","RestrictForCodeCompletion":true,"Scopes":[""], "AnyScope":false}]