subject:"\[PATCH\] D77225\: \[clangd\] Support textDocument\/semanticTokens\/edits"

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

2020-04-06 Thread Nathan Ridge via Phabricator via cfe-commits

nridge added a comment.

> Tested in VSCode insiders (with a patched client to enable experimental 
> features).

In case it's useful to others: the required change to the client to opt into 
this appears to be calling `registerProposedFeatures()` on the `LanguageClient` 
object.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D77225/new/

https://reviews.llvm.org/D77225



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

2020-04-02 Thread Sam McCall via Phabricator via cfe-commits

This revision was automatically updated to reflect the committed changes.
Closed by commit rG9e3063eaceec: [clangd] Support 
textDocument/semanticTokens/edits (authored by sammccall).

Changed prior to commit:
  https://reviews.llvm.org/D77225?vs=254226=254536#toc

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D77225/new/

https://reviews.llvm.org/D77225

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/Protocol.cpp
  clang-tools-extra/clangd/Protocol.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/initialize-params.test
  clang-tools-extra/clangd/test/semantic-tokens.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -14,8 +14,10 @@
 #include "TestFS.h"
 #include "TestTU.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include 
 
@@ -23,6 +25,9 @@
 namespace clangd {
 namespace {
 
+using testing::IsEmpty;
+using testing::SizeIs;
+
 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
 
@@ -720,25 +725,29 @@
   ASSERT_EQ(Counter.Count, 1);
 }
 
+// Ranges are highlighted as variables, unless highlighted as $Function etc.
+std::vector tokens(llvm::StringRef MarkedText) {
+  Annotations A(MarkedText);
+  std::vector Results;
+  for (const Range& R : A.ranges())
+Results.push_back({HighlightingKind::Variable, R});
+  for (unsigned I = 0; I < static_cast(HighlightingKind::LastKind); ++I) {
+HighlightingKind Kind = static_cast(I);
+for (const Range& R : A.ranges(llvm::to_string(Kind)))
+  Results.push_back({Kind, R});
+  }
+  llvm::sort(Results);
+  return Results;
+}
+
 TEST(SemanticHighlighting, toSemanticTokens) {
-  auto CreatePosition = [](int Line, int Character) -> Position {
-Position Pos;
-Pos.line = Line;
-Pos.character = Character;
-return Pos;
-  };
+  auto Results = toSemanticTokens(tokens(R"(
+ [[blah]]
 
-  std::vector Tokens = {
-  {HighlightingKind::Variable,
-   Range{CreatePosition(1, 1), CreatePosition(1, 5)}},
-  {HighlightingKind::Function,
-   Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-   Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  };
+$Function[[big]] [[bang]]
+  )"));
 
-  std::vector Results = toSemanticTokens(Tokens);
-  EXPECT_EQ(Tokens.size(), Results.size());
+  ASSERT_THAT(Results, SizeIs(3));
   EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
   EXPECT_EQ(Results[0].deltaLine, 1u);
   EXPECT_EQ(Results[0].deltaStart, 1u);
@@ -755,6 +764,38 @@
   EXPECT_EQ(Results[2].length, 4u);
 }
 
+TEST(SemanticHighlighting, diffSemanticTokens) {
+  auto Before = toSemanticTokens(tokens(R"(
+[[foo]] [[bar]] [[baz]]
+[[one]] [[two]] [[three]]
+  )"));
+  EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
+
+  auto After = toSemanticTokens(tokens(R"(
+[[foo]] [[hello]] [[world]] [[baz]]
+[[one]] [[two]] [[three]]
+  )"));
+
+  // Replace [bar, baz] with [hello, world, baz]
+  auto Diff = diffTokens(Before, After);
+  ASSERT_THAT(Diff, SizeIs(1));
+  EXPECT_EQ(1u, Diff.front().startToken);
+  EXPECT_EQ(2u, Diff.front().deleteTokens);
+  ASSERT_THAT(Diff.front().tokens, SizeIs(3));
+  // hello
+  EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
+  EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[0].length);
+  // world
+  EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[1].length);
+  // baz
+  EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
+  EXPECT_EQ(3u, Diff.front().tokens[2].length);
+}
+
 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
   auto CreatePosition = [](int Line, int Character) -> Position {
 Position Pos;
Index: clang-tools-extra/clangd/test/semantic-tokens.test
===
--- clang-tools-extra/clangd/test/semantic-tokens.test
+++ clang-tools-extra/clangd/test/semantic-tokens.test
@@ -6,8 +6,13 @@
   "semanticTokens":{"dynamicRegistration":true}
 
 ---
-{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}}

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

2020-04-02 Thread Haojian Wu via Phabricator via cfe-commits

hokein accepted this revision.
hokein added a comment.
This revision is now accepted and ready to land.

this is neat, and the new semantic highlighting protocol is much nicer than the 
old one.




Comment at: clang-tools-extra/clangd/ClangdLSPServer.cpp:1245
+static void increment(std::string ) {
+  for (char  : llvm::reverse(S)) {
+if (C != '9') {

nit: add assert(C >= '0' && C <= '9').



Comment at: clang-tools-extra/clangd/SemanticHighlighting.cpp:605
+diffTokens(llvm::ArrayRef Old,
+   llvm::ArrayRef New) {
+  // For now, just replace everything from the first-last modification.

nit: assert Old and New are sorted.



Comment at: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp:734
+Results.push_back({HighlightingKind::Variable, R});
+  for (unsigned I = 0; I < static_cast(HighlightingKind::LastKind); 
++I) {
+HighlightingKind Kind = static_cast(I);

should be `<=`?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D77225/new/

https://reviews.llvm.org/D77225



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

2020-04-01 Thread Sam McCall via Phabricator via cfe-commits

sammccall created this revision.
sammccall added a reviewer: hokein.
Herald added subscribers: cfe-commits, usaxena95, kadircet, arphaman, mgrang, 
jkorous, MaskRay, ilya-biryukov.
Herald added a project: clang.

This returns incremental highlights as a set of edits against the
previous highlights.

Server-side, we compute the full set of highlights, this just saves
wire-format size.

For now, the diff used is trivial: everything from the first change to
the last change is sent as a single edit.

The wire format is grungy - the replacement offset/length refer to
positions in the encoded array instead of the logical list of tokens.
We use token-oriented structs and translating to LSP forms when serializing.
This departs from LSP (but is consistent with semanticTokens today).

Tested in VSCode insiders (with a patched client to enable experimental
features).


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D77225

Files:
  clang-tools-extra/clangd/ClangdLSPServer.cpp
  clang-tools-extra/clangd/ClangdLSPServer.h
  clang-tools-extra/clangd/Protocol.cpp
  clang-tools-extra/clangd/Protocol.h
  clang-tools-extra/clangd/SemanticHighlighting.cpp
  clang-tools-extra/clangd/SemanticHighlighting.h
  clang-tools-extra/clangd/test/initialize-params.test
  clang-tools-extra/clangd/test/semantic-tokens.test
  clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

Index: clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
===
--- clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -14,8 +14,10 @@
 #include "TestFS.h"
 #include "TestTU.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include 
 
@@ -23,6 +25,9 @@
 namespace clangd {
 namespace {
 
+using testing::IsEmpty;
+using testing::SizeIs;
+
 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
 
@@ -720,25 +725,29 @@
   ASSERT_EQ(Counter.Count, 1);
 }
 
+// Ranges are highlighted as variables, unless highlighted as $Function etc.
+std::vector tokens(llvm::StringRef MarkedText) {
+  Annotations A(MarkedText);
+  std::vector Results;
+  for (const Range& R : A.ranges())
+Results.push_back({HighlightingKind::Variable, R});
+  for (unsigned I = 0; I < static_cast(HighlightingKind::LastKind); ++I) {
+HighlightingKind Kind = static_cast(I);
+for (const Range& R : A.ranges(llvm::to_string(Kind)))
+  Results.push_back({Kind, R});
+  }
+  llvm::sort(Results);
+  return Results;
+}
+
 TEST(SemanticHighlighting, toSemanticTokens) {
-  auto CreatePosition = [](int Line, int Character) -> Position {
-Position Pos;
-Pos.line = Line;
-Pos.character = Character;
-return Pos;
-  };
+  auto Results = toSemanticTokens(tokens(R"(
+ [[blah]]
 
-  std::vector Tokens = {
-  {HighlightingKind::Variable,
-   Range{CreatePosition(1, 1), CreatePosition(1, 5)}},
-  {HighlightingKind::Function,
-   Range{CreatePosition(3, 4), CreatePosition(3, 7)}},
-  {HighlightingKind::Variable,
-   Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
-  };
+$Function[[big]] [[bang]]
+  )"));
 
-  std::vector Results = toSemanticTokens(Tokens);
-  EXPECT_EQ(Tokens.size(), Results.size());
+  ASSERT_THAT(Results, SizeIs(3));
   EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
   EXPECT_EQ(Results[0].deltaLine, 1u);
   EXPECT_EQ(Results[0].deltaStart, 1u);
@@ -755,6 +764,38 @@
   EXPECT_EQ(Results[2].length, 4u);
 }
 
+TEST(SemanticHighlighting, diffSemanticTokens) {
+  auto Before = toSemanticTokens(tokens(R"(
+[[foo]] [[bar]] [[baz]]
+[[one]] [[two]] [[three]]
+  )"));
+  EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
+
+  auto After = toSemanticTokens(tokens(R"(
+[[foo]] [[hello]] [[world]] [[baz]]
+[[one]] [[two]] [[three]]
+  )"));
+
+  // Replace [bar, baz] with [hello, world, baz]
+  auto Diff = diffTokens(Before, After);
+  ASSERT_THAT(Diff, SizeIs(1));
+  EXPECT_EQ(1u, Diff.front().startToken);
+  EXPECT_EQ(2u, Diff.front().deleteTokens);
+  ASSERT_THAT(Diff.front().tokens, SizeIs(3));
+  // hello
+  EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
+  EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[0].length);
+  // world
+  EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[1].length);
+  // baz
+  EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
+  EXPECT_EQ(3u, Diff.front().tokens[2].length);
+}
+
 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
   auto CreatePosition = [](int Line, int Character) ->

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

[PATCH] D77225: [clangd] Support textDocument/semanticTokens/edits

4 matches

Site Navigation

Mail list logo

Footer information