[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi closed this revision.
akyrtzi added a comment.

b4c83a13f664582015ea22924b9a0c6290d41f5b 



Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi added a comment.

Thank you @jansvoboda11 for reviewing and helping me qualify the changes! ‍♂️

If I receive no further comments I will commit the changes on Thursday; if 
anyone wants some more time to take a look just let me know.
I intend to commit in 2 commits, the first patch as one commit and the next 
three as a second commit; this is to ensure there's no commit where there's a 
performance regression in dependency scanning.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Jan Svoboda via Phabricator via cfe-commits]

jansvoboda11 accepted this revision.
jansvoboda11 added a comment.
This revision is now accepted and ready to land.

LGTM.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi updated this revision to Diff 431151.
akyrtzi added a comment.

Update due to source change in the previous patch 
(https://reviews.llvm.org/D125487)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

Files:
  clang/include/clang/Lex/Lexer.h
  clang/include/clang/Lex/Preprocessor.h
  clang/include/clang/Lex/PreprocessorOptions.h
  clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
  clang/lib/Lex/Lexer.cpp
  clang/lib/Lex/PPDirectives.cpp
  clang/lib/Lex/PPLexerChange.cpp
  clang/lib/Lex/Preprocessor.cpp
  clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -186,6 +186,17 @@
   // filesystem.
   FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
   ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
+
+  llvm::IntrusiveRefCntPtr LocalDepFS =
+  DepFS;
+  ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
+  [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
+  -> Optional> {
+if (llvm::ErrorOr Entry =
+LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
+  return Entry->getDirectiveTokens();
+return None;
+  };
 }
 
 // Create the dependency collector that will collect the produced
Index: clang/lib/Lex/Preprocessor.cpp
===
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -377,7 +377,9 @@
 
 void Preprocessor::recomputeCurLexerKind() {
   if (CurLexer)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = CurLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
   else if (CurTokenLexer)
 CurLexerKind = CLK_TokenLexer;
   else
@@ -640,6 +642,9 @@
 case CLK_CachingLexer:
   CachingLex(Tok);
   break;
+case CLK_DependencyDirectivesLexer:
+  CurLexer->LexDependencyDirectiveToken(Tok);
+  break;
 case CLK_LexAfterModuleImport:
   LexAfterModuleImport(Tok);
   break;
@@ -901,6 +906,9 @@
   CachingLex(Result);
   ReturnedToken = true;
   break;
+case CLK_DependencyDirectivesLexer:
+  ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result);
+  break;
 case CLK_LexAfterModuleImport:
   ReturnedToken = LexAfterModuleImport(Result);
   break;
Index: clang/lib/Lex/PPLexerChange.cpp
===
--- clang/lib/Lex/PPLexerChange.cpp
+++ clang/lib/Lex/PPLexerChange.cpp
@@ -91,8 +91,19 @@
 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
   }
 
-  EnterSourceFileWithLexer(
-  new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile), CurDir);
+  Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
+  if (getPreprocessorOpts().DependencyDirectivesForFile &&
+  FID != PredefinesFileID) {
+if (Optional File = SourceMgr.getFileEntryRefForID(FID)) {
+  if (Optional>
+  DepDirectives =
+  getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
+TheLexer->DepDirectives = *DepDirectives;
+  }
+}
+  }
+
+  EnterSourceFileWithLexer(TheLexer, CurDir);
   return false;
 }
 
@@ -110,7 +121,9 @@
   CurDirLookup = CurDir;
   CurLexerSubmodule = nullptr;
   if (CurLexerKind != CLK_LexAfterModuleImport)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = TheLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
 
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks && !CurLexer->Is_PragmaLexer) {
Index: clang/lib/Lex/PPDirectives.cpp
===
--- clang/lib/Lex/PPDirectives.cpp
+++ clang/lib/Lex/PPDirectives.cpp
@@ -426,29 +426,40 @@
   Token Tok;
   SourceLocation endLoc;
   while (true) {
-CurLexer->Lex(Tok);
+if (CurLexer->isDependencyDirectivesLexer()) {
+  CurLexer->LexDependencyDirectiveTokenWhileSkipping(Tok);
+} else {
+  while (true) {
+CurLexer->Lex(Tok);
 
-if (Tok.is(tok::code_completion)) {
-  setCodeCompletionReached();
-  if (CodeComplete)
-CodeComplete->CodeCompleteInConditionalExclusion();
-  continue;
-}
+if (Tok.is(tok::code_completion)) {
+  setCodeCompletionReached();
+  if (CodeComplete)
+CodeComplete->CodeCompleteInConditionalExclusion();
+  continue;
+}
 
-// If this is the end 

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi added a comment.

In D125488#3510329 , @dexonsmith 
wrote:

> Seems unfortunate to have a temporary regression in the commit stack, since 
> then you can't push incrementally (or bisect). Can the prior patch leave 
> behind the feature in the DependencyFilesystem, and this patch delete it now 
> that clang-scan-deps doesn't depend on it for performance? (Or ignore me if 
> I'm still not understanding...)

I think creating an intermediate commit state where it's both doing lexing for 
tokens //and// source minimization is not worth the trouble.
When committing I could merge the last 2 patches together into one commit (or 
all 4 of them into a commit), so there's no commit state where depscan 
performance is regressed.
I mostly separated them like this for reviewing convenience.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Duncan P. N. Exon Smith via Phabricator via cfe-commits]

dexonsmith added a comment.

In D125488#3510320 , @akyrtzi wrote:

> In D125488#3510297 , @dexonsmith 
> wrote:
>
>> [To be clear, my question was because I don't see this patch deleting the 
>> code path that minimizes / saves-minimized sources. Can/should we delete the 
>> "minimize sources" code path?]
>
> Oh, this is removed in the prior patch in the review stack 
> (https://reviews.llvm.org/D125487)

Seems unfortunate to have a temporary regression in the commit stack, since 
then you can't push incrementally (or bisect). Can the prior patch leave behind 
the feature in the DependencyFilesystem, and this patch delete it now that 
clang-scan-deps doesn't depend on it for performance? (Or ignore me if I'm 
still not understanding...)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi added a comment.

In D125488#3510297 , @dexonsmith 
wrote:

> [To be clear, my question was because I don't see this patch deleting the 
> code path that minimizes / saves-minimized sources. Can/should we delete the 
> "minimize sources" code path?]

Oh, this is removed in the prior patch in the review stack 
(https://reviews.llvm.org/D125487)

> I like the direction of trying to remove FSStatCache. I think stat/content 
> caching belongs at the VFS level so DepFS seems like a better starting point 
> (maybe generalized a bit). Note that DepFS isn't just amortizing `stat` cost, 
> it's also avoiding reopening / `mmap`ing / `mempcy`ing files.

That's a good point, so maybe the direction to go is to generalize the DepFS 
caching mechanism.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Duncan P. N. Exon Smith via Phabricator via cfe-commits]

dexonsmith added a comment.

In D125488#3510265 , @akyrtzi wrote:

> In D125488#3510214 , @dexonsmith 
> wrote:
>
>> Is there code in DepFS that can/should be deleted as part of this patch, or 
>> in a follow-up, or is it still around as an option?

[To be clear, my question was because I don't see this patch deleting the code 
path that minimizes / saves-minimized sources. Can/should we delete the 
"minimize sources" code path?]

> After these changes, with DepFS we are using its multi-threading sharding 
> technique to cache file `stat`s and the source file directive scanning 
> results. We could use multi-threading sharding only to cache source file 
> directive scanning results, and get rid of `DepFS` altogether, but then I 
> don't see a good way to cache the file `stat`s as well, unless we want to try 
> to re-use the `FileManager` across depscan instances and make its `stat` 
> caching as efficient as DepFS (maybe by generalizing the multi-threading 
> sharding technique and using it in `FileManager`).
>
> There's also `FileSystemStatCache` which seems like a leftover right now, but 
> we could enhance it and have it shared by individual `FileManager` instances 
> (instead of sharing the same `FileManager` in depscan instances).
>
> What do you think?

FWIW, I don't think we should be sharing FileManager at all, since it has state 
that makes sharing it unsound.

I like the direction of trying to remove FSStatCache. I think stat/content 
caching belongs at the VFS level so DepFS seems like a better starting point 
(maybe generalized a bit). Note that DepFS isn't just amortizing `stat` cost, 
it's also avoiding reopening / `mmap`ing / `mempcy`ing files.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi added a comment.

In D125488#3510214 , @dexonsmith 
wrote:

> Is there code in DepFS that can/should be deleted as part of this patch, or 
> in a follow-up, or is it still around as an option?

After these changes, with DepFS we are using its multi-threading sharding 
technique to cache file `stat`s and the source file directive scanning results. 
We could use multi-threading sharding only to cache source file directive 
scanning results, and get rid of `DepFS` altogether, but then I don't see a 
good way to cache the file `stat`s as well, unless we want to try to re-use the 
`FileManager` across depscan instances and make its `stat` caching as efficient 
as DepFS (maybe by generalizing the multi-threading sharding technique and 
using it in `FileManager`).

There's also `FileSystemStatCache` which seems like a leftover right now, but 
we could enhance it and have it shared by individual `FileManager` instances 
(instead of sharing the same `FileManager` in depscan instances).

What do you think?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Duncan P. N. Exon Smith via Phabricator via cfe-commits]

dexonsmith added a comment.

Awesome to see this! Looking forward to the next step (using this in normal 
preprocessing!).

> after this change, we don't minimize sources and pass them in place of the 
> real sources

Is there code in DepFS that can/should be deleted as part of this patch, or in 
a follow-up, or is it still around as an option?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi updated this revision to Diff 429033.
akyrtzi added a comment.

Added

  #include "clang/Basic/FileEntry.h"

in `PreprocessorOptions.h` to accommodate the modules build.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D125488/new/

https://reviews.llvm.org/D125488

Files:
  clang/include/clang/Lex/Lexer.h
  clang/include/clang/Lex/Preprocessor.h
  clang/include/clang/Lex/PreprocessorOptions.h
  clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
  clang/lib/Lex/Lexer.cpp
  clang/lib/Lex/PPDirectives.cpp
  clang/lib/Lex/PPLexerChange.cpp
  clang/lib/Lex/Preprocessor.cpp
  clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -200,6 +200,17 @@
   // filesystem.
   FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
   ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
+
+  llvm::IntrusiveRefCntPtr LocalDepFS =
+  DepFS;
+  ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
+  [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
+  -> Optional> {
+if (llvm::ErrorOr Entry =
+LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
+  return Entry->getDirectiveTokens();
+return None;
+  };
 }
 
 // Create the dependency collector that will collect the produced
Index: clang/lib/Lex/Preprocessor.cpp
===
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -377,7 +377,9 @@
 
 void Preprocessor::recomputeCurLexerKind() {
   if (CurLexer)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = CurLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
   else if (CurTokenLexer)
 CurLexerKind = CLK_TokenLexer;
   else
@@ -640,6 +642,9 @@
 case CLK_CachingLexer:
   CachingLex(Tok);
   break;
+case CLK_DependencyDirectivesLexer:
+  CurLexer->LexDependencyDirectiveToken(Tok);
+  break;
 case CLK_LexAfterModuleImport:
   LexAfterModuleImport(Tok);
   break;
@@ -901,6 +906,9 @@
   CachingLex(Result);
   ReturnedToken = true;
   break;
+case CLK_DependencyDirectivesLexer:
+  ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result);
+  break;
 case CLK_LexAfterModuleImport:
   ReturnedToken = LexAfterModuleImport(Result);
   break;
Index: clang/lib/Lex/PPLexerChange.cpp
===
--- clang/lib/Lex/PPLexerChange.cpp
+++ clang/lib/Lex/PPLexerChange.cpp
@@ -91,8 +91,19 @@
 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
   }
 
-  EnterSourceFileWithLexer(
-  new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile), CurDir);
+  Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
+  if (getPreprocessorOpts().DependencyDirectivesForFile &&
+  FID != PredefinesFileID) {
+if (Optional File = SourceMgr.getFileEntryRefForID(FID)) {
+  if (Optional>
+  DepDirectives =
+  getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
+TheLexer->DepDirectives = *DepDirectives;
+  }
+}
+  }
+
+  EnterSourceFileWithLexer(TheLexer, CurDir);
   return false;
 }
 
@@ -110,7 +121,9 @@
   CurDirLookup = CurDir;
   CurLexerSubmodule = nullptr;
   if (CurLexerKind != CLK_LexAfterModuleImport)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = TheLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
 
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks && !CurLexer->Is_PragmaLexer) {
Index: clang/lib/Lex/PPDirectives.cpp
===
--- clang/lib/Lex/PPDirectives.cpp
+++ clang/lib/Lex/PPDirectives.cpp
@@ -426,29 +426,40 @@
   Token Tok;
   SourceLocation endLoc;
   while (true) {
-CurLexer->Lex(Tok);
+if (CurLexer->isDependencyDirectivesLexer()) {
+  CurLexer->LexDependencyDirectiveTokenWhileSkipping(Tok);
+} else {
+  while (true) {
+CurLexer->Lex(Tok);
 
-if (Tok.is(tok::code_completion)) {
-  setCodeCompletionReached();
-  if (CodeComplete)
-CodeComplete->CodeCompleteInConditionalExclusion();
-  continue;
-}
+if (Tok.is(tok::code_completion)) {
+  setCodeCompletionReached();
+  if (CodeComplete)
+CodeComplete->CodeCompleteInConditionalExclusion();
+  continue;
+}
 
-// 

[PATCH] D125488: [Preprocessor] Make the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`

[Argyrios Kyrtzidis via Phabricator via cfe-commits]

akyrtzi created this revision.
Herald added a project: All.
akyrtzi requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.

Depends on D125487 

This is 4/4 of a series of patches, bringing the following benefits:

- Full access to the preprocessor state during dependency scanning. E.g. a 
component can see what includes were taken and where they were located in the 
actual sources.
- Improved performance for dependency scanning. Measurements with a 
release+thin-LTO build shows ~ -11% reduction in wall time.
- Opportunity to use dependency scanning lexing to speed-up skipping of 
excluded conditional blocks during normal preprocessing (as follow-up, not part 
of this patch).

For normal preprocessing measurements show differences are below the noise 
level.

Since, after this change, we don't minimize sources and pass them in place of 
the real sources, `DependencyScanningFilesystem` is not technically necessary, 
but it has valuable performance benefits for caching file `stat`s along with 
the results of scanning the sources. So the setup of using the 
`DependencyScanningFilesystem` during a dependency scan remains.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D125488

Files:
  clang/include/clang/Lex/Lexer.h
  clang/include/clang/Lex/Preprocessor.h
  clang/include/clang/Lex/PreprocessorOptions.h
  clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
  clang/lib/Lex/Lexer.cpp
  clang/lib/Lex/PPDirectives.cpp
  clang/lib/Lex/PPLexerChange.cpp
  clang/lib/Lex/Preprocessor.cpp
  clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -200,6 +200,17 @@
   // filesystem.
   FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
   ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
+
+  llvm::IntrusiveRefCntPtr LocalDepFS =
+  DepFS;
+  ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
+  [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
+  -> Optional> {
+if (llvm::ErrorOr Entry =
+LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
+  return Entry->getDirectiveTokens();
+return None;
+  };
 }
 
 // Create the dependency collector that will collect the produced
Index: clang/lib/Lex/Preprocessor.cpp
===
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -377,7 +377,9 @@
 
 void Preprocessor::recomputeCurLexerKind() {
   if (CurLexer)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = CurLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
   else if (CurTokenLexer)
 CurLexerKind = CLK_TokenLexer;
   else
@@ -640,6 +642,9 @@
 case CLK_CachingLexer:
   CachingLex(Tok);
   break;
+case CLK_DependencyDirectivesLexer:
+  CurLexer->LexDependencyDirectiveToken(Tok);
+  break;
 case CLK_LexAfterModuleImport:
   LexAfterModuleImport(Tok);
   break;
@@ -901,6 +906,9 @@
   CachingLex(Result);
   ReturnedToken = true;
   break;
+case CLK_DependencyDirectivesLexer:
+  ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result);
+  break;
 case CLK_LexAfterModuleImport:
   ReturnedToken = LexAfterModuleImport(Result);
   break;
Index: clang/lib/Lex/PPLexerChange.cpp
===
--- clang/lib/Lex/PPLexerChange.cpp
+++ clang/lib/Lex/PPLexerChange.cpp
@@ -91,8 +91,19 @@
 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
   }
 
-  EnterSourceFileWithLexer(
-  new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile), CurDir);
+  Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
+  if (getPreprocessorOpts().DependencyDirectivesForFile &&
+  FID != PredefinesFileID) {
+if (Optional File = SourceMgr.getFileEntryRefForID(FID)) {
+  if (Optional>
+  DepDirectives =
+  getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
+TheLexer->DepDirectives = *DepDirectives;
+  }
+}
+  }
+
+  EnterSourceFileWithLexer(TheLexer, CurDir);
   return false;
 }
 
@@ -110,7 +121,9 @@
   CurDirLookup = CurDir;
   CurLexerSubmodule = nullptr;
   if (CurLexerKind != CLK_LexAfterModuleImport)
-CurLexerKind = CLK_Lexer;
+CurLexerKind = TheLexer->isDependencyDirectivesLexer()
+   ? CLK_DependencyDirectivesLexer
+   : CLK_Lexer;
 
   // Notify